Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-6.5/pull_cdc_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Still waiting to schedule task All nodes of label β€˜pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-1221c’ are offline Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-1221c-jrr88 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8e49e0977ad7e57e5aa7a6fe8fba0ab0a3bf0b30" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-1221c" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" First time build. Skipping changelog. [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 20 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-list --no-walk 618760b8f23481032f6d0010db684d46f840e8e1 # timeout=10 + printenv PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=7f10e61f-b3d5-4268-8708-8b828e862ee0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-1221c GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-de6e69a8ea75a42ea3ef5bc05e88104f5785c12a567deb6ce7101124d13f2fcf NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-1221c pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE="" GOARCH="amd64" GOBIN="" GOCACHE="/home/jenkins/.cache/go-build" GOENV="/home/jenkins/.config/go/env" GOEXE="" GOEXPERIMENT="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="linux" GOINSECURE="" GOMODCACHE="/go/pkg/mod" GONOPROXY="" GONOSUMDB="" GOOS="linux" GOPATH="/go" GOPRIVATE="" GOPROXY="http://goproxy.apps.svc,https://proxy.golang.org,direct" GOROOT="/usr/local/go" GOSUMDB="sum.golang.org" GOTMPDIR="" GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" GOVCS="" GOVERSION="go1.19.13" GCCGO="gccgo" GOAMD64="v1" AR="ar" CC="gcc" CXX="g++" CGO_ENABLED="1" GOMOD="/dev/null" GOWORK="" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build4067258014=/tmp/go-build -gno-record-gcc-switches" + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xl75v bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 65107 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: cf39297c7b34dce8 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 25 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 26 10:47:33 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10959/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10959/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10959/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10959/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/sinkv2/eventsink/txn/txn_sink.go, cdc/sinkv2/eventsink/txn/worker.go, pkg/causality/conflict_detector.go, pkg/causality/internal/node.go, pkg/causality/internal/node_test.go, pkg/causality/internal/slots.go, pkg/causality/internal/slots_test.go, pkg/causality/tests/driver.go, pkg/causality/tests/worker.go, pkg/causality/txn_cache.go, pkg/causality/txn_cache_test.go, pkg/causality/worker.go] [Pipeline] echo diff file not matched: cdc/sinkv2/eventsink/txn/txn_sink.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-986ebf7-c3f8823) 199550976 bytes in 0.87 secs (229033849 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/.git/ .git HEAD is now at c3f88235b sink(ticdc): limit the number of transactions cached in a mysql worker (#10892) POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-6.5 -> origin/release-6.5 = [up to date] refs/pull/10959/head -> origin/pr/10959/head Previous HEAD position was c3f88235b sink(ticdc): limit the number of transactions cached in a mysql worker (#10892) HEAD is now at 986ebf783 *(ticdc): Revert pr 10780 for release-6.5 (#10882) 🚧 Checkouting to base SHA:986ebf783d31f75a3395550cfe8a375ede4d6671... HEAD is now at 986ebf783 *(ticdc): Revert pr 10780 for release-6.5 (#10882) βœ… Checked. πŸŽ‰ 🧾 HEAD info: 986ebf783d31f75a3395550cfe8a375ede4d6671 986ebf783 *(ticdc): Revert pr 10780 for release-6.5 (#10882) 85535a21b ticdc: fix ddl `exchange partition xxxx with validation` to deal with `with validation` correctly (#10867) (#10868) 06ec4a64e filter (ticdc): fix synpoint table is replicated in BDR mode (#10587) (#10628) 🚧 Pre-merge heads of pull requests to base SHA: 986ebf783d31f75a3395550cfe8a375ede4d6671 ... Updating 986ebf783..c3f88235b Fast-forward cdc/sinkv2/eventsink/txn/txn_sink.go | 16 ++-- cdc/sinkv2/eventsink/txn/worker.go | 42 +++------ pkg/causality/conflict_detector.go | 106 ++++++++++++--------- pkg/causality/internal/node.go | 177 ++++++++++++++++++----------------- pkg/causality/internal/node_test.go | 95 ++++++++++--------- pkg/causality/internal/slots.go | 8 +- pkg/causality/internal/slots_test.go | 16 ++-- pkg/causality/tests/driver.go | 12 ++- pkg/causality/tests/worker.go | 29 ++---- pkg/causality/txn_cache.go | 142 ++++++++++++++++++++++++++++ pkg/causality/txn_cache_test.go | 85 +++++++++++++++++ pkg/causality/worker.go | 29 ------ 12 files changed, 485 insertions(+), 272 deletions(-) create mode 100644 pkg/causality/txn_cache.go create mode 100644 pkg/causality/txn_cache_test.go delete mode 100644 pkg/causality/worker.go 🧾 Pre-merged result: c3f88235bc30ad0173bd585b87a683ca1cf9280f c3f88235b sink(ticdc): limit the number of transactions cached in a mysql worker (#10892) 986ebf783 *(ticdc): Revert pr 10780 for release-6.5 (#10882) 85535a21b ticdc: fix ddl `exchange partition xxxx with validation` to deal with `with validation` correctly (#10867) (#10868) βœ… Pre merged πŸŽ‰ βœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-986ebf7-c3f8823 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + chmod +x ../scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh + /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/scripts/pingcap/tiflow/release-6.5/ticdc_download_integration_test_binaries.sh Download binaries from branch release-6.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 817 0 --:--:-- --:--:-- --:--:-- 820 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 774 0 --:--:-- --:--:-- --:--:-- 773 100 41 100 41 0 0 772 0 --:--:-- --:--:-- --:--:-- 773 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2047 0 --:--:-- --:--:-- --:--:-- 2157 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 942 0 --:--:-- --:--:-- --:--:-- 953 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/7357987916b70e4721952d8998428db7bfb78adc/centos7/tidb-server.tar.gz 2024-04-26 18:48:08 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/7357987916b70e4721952d8998428db7bfb78adc/centos7/tidb-server.tar.gz [501213050/501213050] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/0ecafcb3657b51a1ae14f7ca79d5aa6175d70371/centos7/pd-server.tar.gz 2024-04-26 18:48:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/0ecafcb3657b51a1ae14f7ca79d5aa6175d70371/centos7/pd-server.tar.gz [140291455/140291455] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/172a25c1447d3cca370becc0cd7ceda40cda65d5/centos7/tikv-server.tar.gz 2024-04-26 18:49:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/172a25c1447d3cca370becc0cd7ceda40cda65d5/centos7/tikv-server.tar.gz [723187965/723187965] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/0be447c1167454e092fadf228aa2ab11cef3f6aa/centos7/tiflash.tar.gz 2024-04-26 18:49:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-6.5/0be447c1167454e092fadf228aa2ab11cef3f6aa/centos7/tiflash.tar.gz [205557291/205557291] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-04-26 18:49:25 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-04-26 18:49:28 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-04-26 18:49:29 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-04-26 18:49:30 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-00998a9a_linux-amd64.tar.gz 2024-04-26 18:49:31 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-00998a9a_linux-amd64.tar.gz [32158802/32158802] -> "tmp/sync_diff_inspector.tar.gz" [1] total 1.3G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 18 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:49 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:49 jq lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Mar 30 22:23 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Mar 30 22:23 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Mar 30 22:02 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M Mar 30 22:17 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 14:26 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M Apr 26 14:26 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M Apr 26 14:26 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M Apr 26 14:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M Apr 26 14:26 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M Apr 26 14:26 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M Apr 26 14:26 stores-dump -rwxr-xr-x. 1 jenkins jenkins 80M Feb 28 2022 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M Apr 17 23:20 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M Mar 30 22:23 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M Apr 25 18:01 tikv-server Download SUCCESS + ls -alh ./bin/ total 1.3G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 18 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:49 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:49 jq lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Mar 30 22:23 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Mar 30 22:23 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Mar 30 22:02 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M Mar 30 22:17 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 14:26 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M Apr 26 14:26 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M Apr 26 14:26 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M Apr 26 14:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M Apr 26 14:26 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M Apr 26 14:26 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M Apr 26 14:26 stores-dump -rwxr-xr-x. 1 jenkins jenkins 80M Feb 28 2022 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M Apr 17 23:20 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M Mar 30 22:23 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M Apr 25 18:01 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin/minio + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.3G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:49 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:49 jq lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Mar 30 22:23 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Mar 30 22:23 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Mar 30 22:23 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Mar 30 22:02 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M Mar 30 22:17 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 14:26 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M Apr 26 14:26 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M Apr 26 14:26 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M Apr 26 14:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M Apr 26 14:26 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M Apr 26 14:26 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M Apr 26 14:26 stores-dump -rwxr-xr-x. 1 jenkins jenkins 80M Feb 28 2022 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M Apr 17 23:20 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M Mar 30 22:23 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M Apr 25 18:01 tikv-server + ./bin/tidb-server -V Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 + ./bin/tikv-server -V TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v5.4.0-2-g00998a9 Git Commit Hash: 00998a9a4bfd08eb63449186d147b0e6bb1fea5d Git Branch: master UTC Build Time: 2022-02-21 04:15:21 Go Version: go1.16.4 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/cdc-integration-test/rev-986ebf7-c3f8823) 781206016 bytes in 1.47 secs (532624424 bytes/sec) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 746M drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 18 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 162M Apr 26 18:26 cdc -rwxr-xr-x. 1 jenkins jenkins 311M Apr 26 18:26 cdc.test -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_storage_consumer + '[' -f ./bin/cdc ']' + '[' -f ./bin/cdc_kafka_consumer ']' + '[' -f ./bin/cdc.test ']' + ls -alh ./bin total 746M drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 18 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 162M Apr 26 18:26 cdc -rwxr-xr-x. 1 jenkins jenkins 311M Apr 26 18:26 cdc.test -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_storage_consumer + ./bin/cdc version Release Version: v6.5.9-1-gc3f88235b Git Commit Hash: c3f88235bc30ad0173bd585b87a683ca1cf9280f Git Branch: HEAD UTC Build Time: 2024-04-26 10:22:08 Go Version: go version go1.19.12 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-986ebf7-c3f8823 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.0G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:49 . drwxr-sr-x. 18 jenkins jenkins 4.0K Apr 26 18:49 .. -rwxr-xr-x. 1 jenkins jenkins 162M Apr 26 18:26 cdc -rwxr-xr-x. 1 jenkins jenkins 311M Apr 26 18:26 cdc.test -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 137M Apr 26 18:22 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 17M Apr 26 18:49 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:49 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:49 jq lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 18:49 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Apr 26 18:49 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 26 18:49 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Apr 26 18:49 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 18:49 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 26 18:49 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 26 18:49 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 74M Apr 26 18:49 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Apr 26 18:49 minio -rwxr-xr-x. 1 jenkins jenkins 36M Apr 26 18:49 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 20M Apr 26 18:49 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 34M Apr 26 18:49 pd-recover -rwxr-xr-x. 1 jenkins jenkins 95M Apr 26 18:49 pd-server -rwxr-xr-x. 1 jenkins jenkins 23M Apr 26 18:49 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 27M Apr 26 18:49 regions-dump -rwxr-xr-x. 1 jenkins jenkins 22M Apr 26 18:49 stores-dump -rwxr-xr-x. 1 jenkins jenkins 80M Apr 26 18:49 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 170M Apr 26 18:49 tidb-server -rwxr-xr-x. 1 jenkins jenkins 297M Apr 26 18:49 tiflash -rwxr-xr-x. 1 jenkins jenkins 304M Apr 26 18:49 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 62.09 secs (37628523 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G0') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G1') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G2') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G3') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G4') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G5') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G6') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G7') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G8') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G9') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G0') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G1') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G2') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G3') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G4') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G5') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G6') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G7') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G8') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G9') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 [Pipeline] node Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sbxwz-27ssm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c3577c1d1a54d6167a3ce68e58c81fdc3ad909d4" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sbxwz" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-978gr-75gz4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a89dc6acead08bc5d3afb7d844f64eb9309ad291" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-978gr" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@aac0dd0; decorates RemoteLauncher[hudson.remoting.Channel@5fbe285c:JNLP4-connect connection from 10.233.86.115/10.233.86.115:36550] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5b518888; decorates RemoteLauncher[hudson.remoting.Channel@4c28e007:JNLP4-connect connection from 10.233.67.213/10.233.67.213:34694] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] node Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-cwxgq-qx4w8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6d760078d097548888b5d66e71fab51f935ff4d8" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-cwxgq" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "feat(tidb): test flashbacktest package (#2942)" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e15047a; decorates RemoteLauncher[hudson.remoting.Channel@6fab4afe:JNLP4-connect connection from 10.233.69.103/10.233.69.103:38396] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rtc8s-x5j1w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "56a1cb93062063f1b1c541ed806ad560ece43997" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rtc8s" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "feat(tidb): test flashbacktest package (#2942)" Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-7qkzt-c4zp8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3eda53f4114a16f2d80e9dfd539a960d29922f9a" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-7qkzt" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-m08r9-0dzck --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ef8a842583a0584994abb51f38f748512b778cc6" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-m08r9" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-lddl1-9rxb9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8bce1e12397a6b7f2f4f333ea3aa5d17a15d9f90" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-lddl1" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Obtained pipelines/pingcap/tiflow/release-6.5/pod-pull_cdc_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 4.33 secs (539731908 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G0 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group0: api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=5add80c6-4f5b-4778-bd2f-6adb6fd2c34c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G0 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sbxwz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sbxwz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-fhf0l-kq0w9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ac79129bc423af890226cd27ba4a67f460de0ddb" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-fhf0l" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@21154de5; decorates RemoteLauncher[hudson.remoting.Channel@1d3516b6:JNLP4-connect connection from 10.233.86.78/10.233.86.78:35930] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@63bdc080; decorates RemoteLauncher[hudson.remoting.Channel@7db64e80:JNLP4-connect connection from 10.233.107.227/10.233.107.227:34540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@22ba142; decorates RemoteLauncher[hudson.remoting.Channel@149e4d8c:JNLP4-connect connection from 10.233.68.177/10.233.68.177:57382] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@131accfe; decorates RemoteLauncher[hudson.remoting.Channel@5de68388:JNLP4-connect connection from 10.233.71.140/10.233.71.140:39066] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Still waiting to schedule task β€˜pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm’ is offline > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Still waiting to schedule task β€˜pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8’ is offline Commit message: "feat(tidb): test flashbacktest package (#2942)" Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-p18xk-dd2w6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7e38710994bd8bc9b3cf534b51c9257de870d7de" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-p18xk" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5pggs-71mp8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d1e06fc1da11dfe9bf2f3461aab099b8d59da1ce" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5pggs" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5w1pm-4hmx4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2263af9dcfe9f9150c6c11798c60fde744e33032" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5w1pm" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 5.34 secs (437813249 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-gp6dd-qbf0c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "dc2b0131f06288055449cd9401ca6ac7720c1b45" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-gp6dd" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-64xwl-n745x Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-jhwnw-vg7mc --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cc643344e6b95578dfa6e7291f64105b39770fbb" jenkins/label-digest: "1a18ac95d7b9f4957db855b6b849b895b6455966" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-64xwl" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-jhwnw" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" operator: "In" values: - "amd64" containers: values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj" - name: "JENKINS_NAME" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" requests: memory: "256Mi" cpu: "100m" volumeMounts: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" requests: memory: "256Mi" cpu: "100m" volumeMounts: securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" name: "workspace-volume" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Still waiting to schedule task β€˜pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq’ is offline Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rjzqq-c3h6j --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "81bb72ed9a692af6ad41e017cdca4220702bb8e2" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rjzqq" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:12.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:12.773 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f60d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:1088, start at 2024-04-26 18:51:12.837914252 +0800 CST m=+4.309889334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:12.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log Agent pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq is provisioned from template pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sn0wf-mpxth --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" runUrl: "job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "dbea5d26ef4c2a4827905404b94769c70c68e017" jenkins/label: "pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sn0wf" name: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.19:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.4.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 17.15 secs (136241841 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G3 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group3: changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=1c37fa2d-10ae-4429-a79d-7c2c80695899 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G3 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-cwxgq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-cwxgq pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5be6486b; decorates RemoteLauncher[hudson.remoting.Channel@5c70d067:JNLP4-connect connection from 10.233.90.208/10.233.90.208:35730] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@46e391cc; decorates RemoteLauncher[hudson.remoting.Channel@7493ddc3:JNLP4-connect connection from 10.233.107.206/10.233.107.206:34068] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6809c786; decorates RemoteLauncher[hudson.remoting.Channel@d76a2cc:JNLP4-connect connection from 10.233.73.108/10.233.73.108:52036] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@72ed699d; decorates RemoteLauncher[hudson.remoting.Channel@3267f433:JNLP4-connect connection from 10.233.69.15/10.233.69.15:56024] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@13659df1; decorates RemoteLauncher[hudson.remoting.Channel@720abd9c:JNLP4-connect connection from 10.233.84.83/10.233.84.83:44358] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4d5a544; decorates RemoteLauncher[hudson.remoting.Channel@4ec729bb:JNLP4-connect connection from 10.233.127.1/10.233.127.1:58340] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@431c823d; decorates RemoteLauncher[hudson.remoting.Channel@e6414e3:JNLP4-connect connection from 10.233.127.224/10.233.127.224:58322] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@21410410; decorates RemoteLauncher[hudson.remoting.Channel@56b21569:JNLP4-connect connection from 10.233.123.227/10.233.123.227:41502] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7df3a0c6; decorates RemoteLauncher[hudson.remoting.Channel@65220097:JNLP4-connect connection from 10.233.67.55/10.233.67.55:48558] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G1 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group1: big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=d380dc47-5e80-4088-aa6d-ea8da24cb971 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G1 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-978gr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-978gr pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Commit message: "feat(tidb): test flashbacktest package (#2942)" Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 8.33 secs (280539722 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:41.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:41.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682127e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:962, start at 2024-04-26 18:51:41.964226342 +0800 CST m=+4.464193414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:41.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading go.uber.org/atomic v1.10.0 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:47.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:47.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682176680006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:1067, start at 2024-04-26 18:51:46.976573686 +0800 CST m=+4.384923227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:53:46.984 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:46.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:46.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 Starting CDC state checker... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 6.02 secs (388287587 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading cloud.google.com/go v0.105.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 14.71 secs (158798092 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 4.05 secs (576170731 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G5 [Pipeline] sh Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group5: consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=410ae03f-79b8-42f1-898d-2c56ee3fc30a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G5 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rtc8s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rtc8s GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G8 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group8: generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=0c57df0e-c379-428a-b5a3-bb1340c07b16 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G8 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-7qkzt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-7qkzt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G4 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group4: clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=8597a4ef-d058-45cb-8004-c2b1f1b2c126 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G4 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-lddl1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-lddl1 pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G2 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group2: capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=3d83c470-51a5-4b6c-990a-c59ca654166e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G2 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-m08r9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-m08r9 pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:26.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:26.912 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e4100015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:1010, start at 2024-04-26 18:52:26.777907048 +0800 CST m=+4.322656509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:26.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.756 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.756 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:28.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:28.843 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682405600019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:1035, start at 2024-04-26 18:52:28.926649181 +0800 CST m=+4.360888201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:28.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading go.uber.org/multierr v1.8.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:29.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:29.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6824149c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:959, start at 2024-04-26 18:52:29.874332958 +0800 CST m=+4.234296943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:29.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:29.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:29.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682417700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:1062, start at 2024-04-26 18:52:30.074201773 +0800 CST m=+4.447962946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:30.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:30.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:30.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.uber.org/atomic v1.10.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading golang.org/x/net v0.18.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading cloud.google.com/go v0.105.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading golang.org/x/sync v0.5.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 Starting CDC state checker... go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading golang.org/x/time v0.2.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading golang.org/x/net v0.18.0 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading google.golang.org/api v0.103.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 13.81 secs (169209239 bytes/sec) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 [Pipeline] { [Pipeline] cache go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 23.33 secs (100131917 bytes/sec) [Pipeline] { [Pipeline] cache # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 16.94 secs (137923428 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f60d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:1088, start at 2024-04-26 18:51:12.837914252 +0800 CST m=+4.309889334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.902 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 17.50 secs (133518148 bytes/sec) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682127e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:962, start at 2024-04-26 18:51:41.964226342 +0800 CST m=+4.464193414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.987 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f5f4c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:984, start at 2024-04-26 18:51:12.766004498 +0800 CST m=+4.303380190 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c681f60d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:1088, start at 2024-04-26 18:51:12.837914252 +0800 CST m=+4.309889334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:12.902 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:12.821 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:12.821 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682176680006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:1067, start at 2024-04-26 18:51:46.976573686 +0800 CST m=+4.384923227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.070 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:46.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:46.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 8.52 secs (274353273 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682126580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:867, start at 2024-04-26 18:51:41.863417885 +0800 CST m=+4.423784225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682127e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:962, start at 2024-04-26 18:51:41.964226342 +0800 CST m=+4.464193414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:41.987 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:41.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:41.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:54:06 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.2512625128.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:09 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afea41fa-383e-434e-87f0-a06b92b6fc48 {"id":"afea41fa-383e-434e-87f0-a06b92b6fc48","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a5b848e afea41fa-383e-434e-87f0-a06b92b6fc48 /tidb/cdc/default/default/upstream/7362126685575130260 {"id":7362126685575130260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afea41fa-383e-434e-87f0-a06b92b6fc48 {"id":"afea41fa-383e-434e-87f0-a06b92b6fc48","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a5b848e afea41fa-383e-434e-87f0-a06b92b6fc48 /tidb/cdc/default/default/upstream/7362126685575130260 {"id":7362126685575130260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afea41fa-383e-434e-87f0-a06b92b6fc48 {"id":"afea41fa-383e-434e-87f0-a06b92b6fc48","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a5b848e afea41fa-383e-434e-87f0-a06b92b6fc48 /tidb/cdc/default/default/upstream/7362126685575130260 {"id":7362126685575130260,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 12.28 secs (190312275 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6821771c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:972, start at 2024-04-26 18:51:47.023499781 +0800 CST m=+4.518868262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:47.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:47.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682176680006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:1067, start at 2024-04-26 18:51:46.976573686 +0800 CST m=+4.384923227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:54:47.070 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:51:46.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:41:46.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:54:11 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2410824110.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:15 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.24168.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348594647171075 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348594647171075 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:54:15 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.2420924211.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:15 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f32b153a-b014-4b49-aabb-2eeaf71c10a0 {"id":"f32b153a-b014-4b49-aabb-2eeaf71c10a0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a6be58e f32b153a-b014-4b49-aabb-2eeaf71c10a0 /tidb/cdc/default/default/upstream/7362126704882925208 {"id":7362126704882925208,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.24235.out cli changefeed create --start-ts=449348594647171075 '--sink-uri=mysql://normal:123456@127.0.0.1:3306?transaction-atomicity=none' Create changefeed successfully! ID: 6d9e5504-9d63-4503-ab65-56745fd6e4e8 Info: {"upstream_id":7362126704882925208,"namespace":"default","id":"6d9e5504-9d63-4503-ab65-56745fd6e4e8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306?transaction-atomicity=none","create_time":"2024-04-26T18:54:16.015632638+08:00","start_ts":449348594647171075,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348594647171075,"checkpoint_ts":449348594647171075,"checkpoint_time":"2024-04-26 18:54:15.313"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 5.09 secs (458804766 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 7.31 secs (319523720 bytes/sec) [Pipeline] { [Pipeline] cache # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e4100015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:1010, start at 2024-04-26 18:52:26.777907048 +0800 CST m=+4.322656509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.756 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.756 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682405600019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:1035, start at 2024-04-26 18:52:28.926649181 +0800 CST m=+4.360888201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6824149c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:959, start at 2024-04-26 18:52:29.874332958 +0800 CST m=+4.234296943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682417700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:1062, start at 2024-04-26 18:52:30.074201773 +0800 CST m=+4.447962946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:30.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:30.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc) 2336357376 bytes in 13.13 secs (177897502 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G7 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group7: default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=869aa0fd-560e-4a8d-8ce1-b457e66b8b99 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G7 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-fhf0l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-fhf0l pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G14 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group14: sequence simple sink_hang sink_retry sorter split_region Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=93f6720c-28a5-45b8-af70-8e8f679797b2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-p18xk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-p18xk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G12 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group12: owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=6b98c541-8749-45b7-bc2b-56cc6e591e33 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5pggs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5pggs pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G13 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group13: processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=5ca27b96-5ebd-4416-b1a8-aead945b8ee1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5w1pm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-5w1pm pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G11 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group11: multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=65e756c2-54b4-4685-a771-a1540677197e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-gp6dd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-gp6dd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G6 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group6: ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=f1a7c0cd-c7c5-4535-aa23-336f495d3535 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G6 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-jhwnw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-jhwnw pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G10 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group10: many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=696126e0-0528-47a8-84d5-221edb7a7d4d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-64xwl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-64xwl pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G9 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group9: kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=cd1abb63-25a1-4b7f-b178-51e531c4394c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G9 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rjzqq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-rjzqq GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:54:47 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + ln -s /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/bin /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/bin + chmod +x ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh + cp ../scripts/pingcap/tiflow/release-6.5/cdc_run_group.sh tests/integration_tests/ + ./tests/integration_tests/cdc_run_group.sh mysql G15 Total number of valid directories: 99 Number of directories in group0: 7 api_v2 autorandom availability bank batch_add_table batch_update_to_no_batch bdr_mode Number of directories in group1: 7 big_txn canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible canal_json_handle_key_only canal_json_storage_basic canal_json_storage_partition_table Number of directories in group2: 7 capture_session_done_during_task capture_suicide_while_balance_table cdc cdc_server_tips changefeed_auto_stop changefeed_error changefeed_fast_fail Number of directories in group3: 6 changefeed_finish changefeed_pause_resume changefeed_reconstruct changefeed_resume_with_checkpoint_ts charset_gbk cli Number of directories in group4: 6 clustered_index common_1 consistent_partition_table consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs Number of directories in group5: 6 consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Number of directories in group6: 6 ddl_attributes ddl_manager ddl_only_block_related_table ddl_puller_lag ddl_reentrant ddl_sequence Number of directories in group7: 6 default_value drop_many_tables event_filter force_replicate_table foreign_key gc_safepoint Number of directories in group8: 6 generate_column hang_sink_suicide http_api http_api_tls http_proxies kafka_big_messages Number of directories in group9: 6 kafka_compression kafka_messages kafka_sink_error_resume kill_owner_with_ddl kv_client_stream_reconnect lossy_ddl Number of directories in group10: 6 many_pk_or_uk move_table mq_sink_lost_callback multi_capture multi_cdc_cluster multi_changefeed Number of directories in group11: 6 multi_source multi_tables_ddl multi_topics new_ci_collation_with_old_value new_ci_collation_without_old_value open_protocol_handle_key_only Number of directories in group12: 6 owner_remove_table_error owner_resign partition_table processor_err_chan processor_etcd_worker_delay processor_panic Number of directories in group13: 6 processor_resolved_ts_fallback processor_stop_delay region_merge resolve_lock row_format savepoint Number of directories in group14: 6 sequence simple sink_hang sink_retry sorter split_region Number of directories in group15: 6 storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Running tests in group15: storage_cleanup storage_csv_update synced_status syncpoint tidb_mysql_test tiflash Sink type: mysql PROW_JOB_ID=83cf14a7-8260-424a-b494-4e1e22e8d5d3 JENKINS_NODE_COOKIE=f9abbe59-b126-443c-a81b-b1a1e323ea65 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/ GOLANG_VERSION=1.19.13 HOSTNAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=618760b8f23481032f6d0010db684d46f840e8e1 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-6.5/pull_cdc_integration_test","buildid":"1783802139131252739","prowjobid":"83cf14a7-8260-424a-b494-4e1e22e8d5d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-6.5","base_sha":"986ebf783d31f75a3395550cfe8a375ede4d6671","base_link":"https://github.com/pingcap/tiflow/commit/986ebf783d31f75a3395550cfe8a375ede4d6671","pulls":[{"number":10959,"author":"ti-chi-bot","sha":"c3f88235bc30ad0173bd585b87a683ca1cf9280f","title":"sink(ticdc): limit the number of transactions cached in a mysql worker (#10892)","link":"https://github.com/pingcap/tiflow/pull/10959","commit_link":"https://github.com/pingcap/tiflow/pull/10959/commits/c3f88235bc30ad0173bd585b87a683ca1cf9280f","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/../bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-6.5/pull_cdc_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#164 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783802139131252739 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=4643d4c29c55f53fa0349367d7f1bb5ca554ea6ef528c146825b0f8464e2e668 JOB_BASE_NAME=pull_cdc_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=d21e0256601e3552120d928c497b43e4476f0548 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/164/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sn0wf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-14445bb58d9e8a60aad324c951d33f9a46c737d79492f427cf3c92e870758aa3 NODE_LABELS=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq pingcap_tiflow_release-6_5_pull_cdc_integration_test_164-sn0wf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-6.5/job/pull_cdc_integration_test/display/redirect BUILD_NUMBER=164 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.19.13.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri Apr 26 18:54:47 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri Apr 26 18:54:47 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release The 1 times to try to start tidb cluster... Verifying downstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:54:50 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:54:53 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e5a40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:923, start at 2024-04-26 18:52:26.903432175 +0800 CST m=+4.504294361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6823e4100015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:1010, start at 2024-04-26 18:52:26.777907048 +0800 CST m=+4.322656509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:26.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:26.756 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:26.756 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.25653.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682404880003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:930, start at 2024-04-26 18:52:28.836786718 +0800 CST m=+4.361752103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682405600019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:1035, start at 2024-04-26 18:52:28.926649181 +0800 CST m=+4.360888201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:28.950 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:28.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:28.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:54:54 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2541325415.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449348604980887553 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348604980887553 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:54:55 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.2569825700.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:57 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26a4e6c-9b71-4794-b6c1-e93179ca937a {"id":"f26a4e6c-9b71-4794-b6c1-e93179ca937a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13be90 f26a4e6c-9b71-4794-b6c1-e93179ca937a /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26a4e6c-9b71-4794-b6c1-e93179ca937a {"id":"f26a4e6c-9b71-4794-b6c1-e93179ca937a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13be90 f26a4e6c-9b71-4794-b6c1-e93179ca937a /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f26a4e6c-9b71-4794-b6c1-e93179ca937a {"id":"f26a4e6c-9b71-4794-b6c1-e93179ca937a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13be90 f26a4e6c-9b71-4794-b6c1-e93179ca937a /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:58 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 {"id":"2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13bc90 2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 /tidb/cdc/default/default/upstream/7362126875578632061 {"id":7362126875578632061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 {"id":"2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13bc90 2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 /tidb/cdc/default/default/upstream/7362126875578632061 {"id":7362126875578632061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 {"id":"2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13bc90 2cb079ff-d9e1-4f86-a9bd-ad03ff7f9bf0 /tidb/cdc/default/default/upstream/7362126875578632061 {"id":7362126875578632061,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.25763.out cli changefeed create --start-ts=449348604980887553 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: f5ac8c48-ce41-45c8-9f0f-d3121477bc9b Info: {"upstream_id":7362126875578632061,"namespace":"default","id":"f5ac8c48-ce41-45c8-9f0f-d3121477bc9b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:54:58.44343573+08:00","start_ts":449348604980887553,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348604980887553,"checkpoint_ts":449348604980887553,"checkpoint_time":"2024-04-26 18:54:54.733"} PASS [Fri Apr 26 18:54:49 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.2455024552.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:54:52 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/897c6e58-f889-4050-b11b-deefc32cc6dd {"id":"897c6e58-f889-4050-b11b-deefc32cc6dd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a172f0c 897c6e58-f889-4050-b11b-deefc32cc6dd /tidb/cdc/default/default/upstream/7362126559896654840 {"id":7362126559896654840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/897c6e58-f889-4050-b11b-deefc32cc6dd {"id":"897c6e58-f889-4050-b11b-deefc32cc6dd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a172f0c 897c6e58-f889-4050-b11b-deefc32cc6dd /tidb/cdc/default/default/upstream/7362126559896654840 {"id":7362126559896654840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/897c6e58-f889-4050-b11b-deefc32cc6dd {"id":"897c6e58-f889-4050-b11b-deefc32cc6dd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0a172f0c 897c6e58-f889-4050-b11b-deefc32cc6dd /tidb/cdc/default/default/upstream/7362126559896654840 {"id":7362126559896654840,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7362126559896654840,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-04-26T18:54:52.679445201+08:00","start_ts":449348604415967235,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348604415967235,"checkpoint_ts":449348604415967235,"checkpoint_time":"2024-04-26 18:54:52.578"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7362126559896654840, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449348604415967235, "checkpoint_time": "2024-04-26 18:54:52.578", "error": null }' + echo '{ "upstream_id": 7362126559896654840, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449348604415967235, "checkpoint_time": "2024-04-26 18:54:52.578", "error": null }' { "upstream_id": 7362126559896654840, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449348604415967235, "checkpoint_time": "2024-04-26 18:54:52.578", "error": null } ++ echo '{' '"upstream_id":' 7362126559896654840, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449348604415967235, '"checkpoint_time":' '"2024-04-26' '18:54:52.578",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362126559896654840, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449348604415967235, '"checkpoint_time":' '"2024-04-26' '18:54:52.578",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/cenkalti/backoff/v4 v4.0.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.24.0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/locales v0.14.1 go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading github.com/go-logr/logr v1.2.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:57.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:57.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1c50001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:1024, start at 2024-04-26 18:54:57.856474904 +0800 CST m=+4.286730751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:57.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:58.551 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:58.551 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d297c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:1075, start at 2024-04-26 18:54:58.677416525 +0800 CST m=+4.484700001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:58.685 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.655 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.655 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:59.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:59.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d31d00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:1060, start at 2024-04-26 18:54:59.210999328 +0800 CST m=+6.739480592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:56:59.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/sync v0.5.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682415e8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:969, start at 2024-04-26 18:52:29.965920268 +0800 CST m=+4.433915280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682417700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:1062, start at 2024-04-26 18:52:30.074201773 +0800 CST m=+4.447962946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:30.120 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:30.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:30.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:00.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:00.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d45440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:1015, start at 2024-04-26 18:55:00.461086116 +0800 CST m=+4.508163034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:00.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log go: downloading cloud.google.com/go v0.105.0 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/xdg/scram v1.0.3 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 Starting CDC state checker... go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading google.golang.org/grpc v1.51.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/stretchr/testify v1.8.4 go: downloading golang.org/x/time v0.2.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading golang.org/x/text v0.14.0 go: downloading cloud.google.com/go v0.105.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/containerd/cgroups v1.0.4 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682414940010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:877, start at 2024-04-26 18:52:29.879957171 +0800 CST m=+4.327164169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.908 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6824149c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:959, start at 2024-04-26 18:52:29.874332958 +0800 CST m=+4.234296943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:55:29.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:52:29.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:42:29.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 [Fri Apr 26 18:55:03 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.2399723999.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading google.golang.org/api v0.103.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:02.803 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:02.803 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d6c100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:1046, start at 2024-04-26 18:55:02.927267993 +0800 CST m=+4.576411032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:02.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log Starting CDC state checker... go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/VividCortex/ewma v1.2.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:03.219 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:03.219 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d718c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:1006, start at 2024-04-26 18:55:03.288905658 +0800 CST m=+4.444445111 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:03.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.267 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.267 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:04.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:04.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d81900019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1105, start at 2024-04-26 18:55:04.341559791 +0800 CST m=+4.549014764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:04.349 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.341 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.341 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 check diff failed 1-th time, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:55:06 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74dc742b-9e98-42c5-99db-8dfe94a9620f {"id":"74dc742b-9e98-42c5-99db-8dfe94a9620f","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b312a90 74dc742b-9e98-42c5-99db-8dfe94a9620f /tidb/cdc/default/default/upstream/7362126890904409701 {"id":7362126890904409701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74dc742b-9e98-42c5-99db-8dfe94a9620f {"id":"74dc742b-9e98-42c5-99db-8dfe94a9620f","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b312a90 74dc742b-9e98-42c5-99db-8dfe94a9620f /tidb/cdc/default/default/upstream/7362126890904409701 {"id":7362126890904409701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74dc742b-9e98-42c5-99db-8dfe94a9620f {"id":"74dc742b-9e98-42c5-99db-8dfe94a9620f","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b312a90 74dc742b-9e98-42c5-99db-8dfe94a9620f /tidb/cdc/default/default/upstream/7362126890904409701 {"id":7362126890904409701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 664d73f7-2f1e-41dc-8d17-ba5a72000e16 Info: {"upstream_id":7362126890904409701,"namespace":"default","id":"664d73f7-2f1e-41dc-8d17-ba5a72000e16","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:55:07.108681457+08:00","start_ts":449348607343591425,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348607343591425,"checkpoint_ts":449348607343591425,"checkpoint_time":"2024-04-26 18:55:03.746"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:05.152 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:05.152 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e9c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:1078, start at 2024-04-26 18:55:05.149214429 +0800 CST m=+4.336252621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:05.157 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading go.uber.org/zap v1.23.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 check diff successfully go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading go.uber.org/multierr v1.8.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading gopkg.in/yaml.v3 v3.0.1 Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.uber.org/atomic v1.10.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading golang.org/x/sync v0.5.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 Starting CDC state checker... go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/stretchr/testify v1.8.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading golang.org/x/time v0.2.0 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 Starting CDC state checker... go: downloading google.golang.org/grpc v1.51.0 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/stretchr/testify v1.8.4 go: downloading golang.org/x/sync v0.5.0 go: downloading golang.org/x/time v0.2.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading github.com/Shopify/sarama v1.38.1 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:09.349 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:09.349 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dce4c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1287, start at 2024-04-26 18:55:09.244620583 +0800 CST m=+4.388236727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:09.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 ***************** properties ***************** "mysql.db"="big_txn" "threadcount"="1" "readproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" "fieldcount"="100" "requestdistribution"="uniform" "operationcount"="0" "dotransactions"="false" "recordcount"="5000" "workload"="core" "mysql.host"="127.0.0.1" "updateproportion"="0" ********************************************** go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/stathat/consistent v1.0.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting CDC state checker... go: downloading github.com/pingcap/errors v0.11.5-0.20220729040631-518f63d66278 go: downloading github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c go: downloading go.etcd.io/etcd/client/v3 v3.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20230904040343-947701a32c05 go: downloading go.etcd.io/etcd/server/v3 v3.5.2 go: downloading google.golang.org/grpc v1.51.0 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/tikv/pd v1.1.0-beta.0.20220303060546-3695d8164800 go: downloading go.uber.org/atomic v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.4 go: downloading github.com/pingcap/tidb/parser v0.0.0-20240306032306-8cdacb705df6 go: downloading go.uber.org/zap v1.23.0 go: downloading github.com/Shopify/sarama v1.38.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.4 go: downloading github.com/prometheus/client_golang v1.13.0 go: downloading golang.org/x/sync v0.5.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240306032306-8cdacb705df6 go: downloading github.com/xdg/scram v1.0.3 go: downloading github.com/tikv/client-go/v2 v2.0.4-0.20231121073938-194639470f84 go: downloading github.com/aws/aws-sdk-go v1.44.259 go: downloading github.com/pingcap/tidb-tools v7.0.1-0.20231228094724-d6c7fc83380a+incompatible go: downloading github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 go: downloading github.com/pingcap/kvproto v0.0.0-20240112060601-a0e3fbb1eeee go: downloading golang.org/x/time v0.2.0 go: downloading github.com/go-mysql-org/go-mysql v1.6.1-0.20221223014230-81966e15b9c5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/klauspost/compress v1.15.14 go: downloading github.com/pierrec/lz4/v4 v4.1.17 go: downloading cloud.google.com/go/storage v1.27.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.5 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/coreos/go-semver v0.3.0 go: downloading github.com/google/btree v1.1.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.15.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/golang/protobuf v1.5.2 go: downloading google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd go: downloading github.com/containerd/cgroups v1.0.4 go: downloading cloud.google.com/go v0.105.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading github.com/coreos/go-systemd/v22 v22.3.2 go: downloading golang.org/x/net v0.18.0 go: downloading go.uber.org/multierr v1.8.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.1.2 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/prometheus/common v0.37.0 go: downloading github.com/prometheus/procfs v0.8.0 go: downloading google.golang.org/protobuf v1.30.0 go: downloading golang.org/x/text v0.14.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/docker/go-units v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/eapache/go-resiliency v1.3.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading go.etcd.io/bbolt v1.3.6 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 go: downloading go.opentelemetry.io/otel/exporters/otlp v0.20.0 go: downloading go.opentelemetry.io/otel/sdk v0.20.0 go: downloading go.opentelemetry.io/otel v0.20.0 go: downloading go.opentelemetry.io/contrib v0.20.0 go: downloading sigs.k8s.io/yaml v1.2.0 go: downloading cloud.google.com/go/compute/metadata v0.2.1 go: downloading cloud.google.com/go/iam v0.8.0 go: downloading cloud.google.com/go/compute v1.13.0 go: downloading github.com/googleapis/gax-go/v2 v2.7.0 go: downloading golang.org/x/oauth2 v0.2.0 go: downloading google.golang.org/api v0.103.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dustin/go-humanize v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.2 go: downloading github.com/jonboulle/clockwork v0.3.0 go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/gorilla/websocket v1.4.2 go: downloading go.opentelemetry.io/otel/trace v0.20.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.opentelemetry.io/otel/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go: downloading go.opentelemetry.io/otel/sdk/metric v0.20.0 go: downloading go.opentelemetry.io/proto/otlp v0.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/form3tech-oss/jwt-go v3.2.5+incompatible go: downloading go.etcd.io/etcd/client/v2 v2.305.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/BurntSushi/toml v1.2.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/twmb/murmur3 v1.1.3 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/stathat/consistent v1.0.0 go: downloading golang.org/x/tools v0.15.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/iancoleman/strcase v0.2.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cockroachdb/pebble v0.0.0-20220415182917-06c9d3be25b3 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.17 go: downloading github.com/tklauser/go-sysconf v0.3.11 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20211122183932-1daafda22083 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.13.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.19 go: downloading github.com/mattn/go-runewidth v0.0.14 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa go: downloading github.com/pingcap/badger v1.5.1-0.20220314162537-ab58fbf40580 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/rivo/uniseg v0.4.2 go: downloading github.com/tklauser/numcpus v0.6.0 go: downloading github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 go: downloading github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 go: downloading github.com/golang/glog v1.0.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/kr/pretty v0.3.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.4 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/jmespath/go-jmespath v0.4.0 INSERT - Takes(s): 10.0, Count: 1944, OPS: 194.5, Avg(us): 4892, Min(us): 1673, Max(us): 293964, 95th(us): 7000, 99th(us): 93000 INSERT - Takes(s): 20.0, Count: 2394, OPS: 119.7, Avg(us): 8054, Min(us): 1673, Max(us): 394247, 95th(us): 27000, 99th(us): 139000 INSERT - Takes(s): 30.0, Count: 4033, OPS: 134.5, Avg(us): 7135, Min(us): 1673, Max(us): 394247, 95th(us): 32000, 99th(us): 100000 Run finished, takes 35.244376169s INSERT - Takes(s): 35.2, Count: 5000, OPS: 141.9, Avg(us): 6743, Min(us): 1673, Max(us): 394247, 95th(us): 29000, 99th(us): 97000 table big_txn.usertable exists ***************** properties ***************** "threadcount"="10" "mysql.user"="root" "insertproportion"="0" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "fieldcount"="100" "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.db"="consistent_replicate_storage_file" "readproportion"="0" "operationcount"="0" "recordcount"="5000" "readallfields"="true" "requestdistribution"="uniform" ********************************************** pass test: get status table generate_column.t not exists for 1-th check, retry later table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later pass test: health [2024/04/26 18:55:55.274 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] Run finished, takes 3.574933181s INSERT - Takes(s): 3.6, Count: 5000, OPS: 1402.2, Avg(us): 6659, Min(us): 1572, Max(us): 92700, 95th(us): 53000, 99th(us): 84000 table generate_column.t1 exists check diff failed 1-th time, retry later table consistent_replicate_storage_file.usertable exists pass test: changefeed apis table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later [2024/04/26 18:55:58.925 +08:00] [INFO] [cases.go:392] ["check changefeed failed"] [time=0] [info=null] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:55:59 CST 2024] <<<<<< run test case generate_column success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later pass test: delete changefeed apis pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit ++ date + echo '[Fri Apr 26 18:56:02 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Fri Apr 26 18:56:02 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:56:08 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.24200.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348622582808577 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348622582808577 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:56:02 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2425124253.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:56:05 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcde575b-a5de-4713-9bb9-84c1c22ccb71 {"id":"dcde575b-a5de-4713-9bb9-84c1c22ccb71","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b323418 dcde575b-a5de-4713-9bb9-84c1c22ccb71 /tidb/cdc/default/default/upstream/7362126888831009041 {"id":7362126888831009041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcde575b-a5de-4713-9bb9-84c1c22ccb71 {"id":"dcde575b-a5de-4713-9bb9-84c1c22ccb71","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b323418 dcde575b-a5de-4713-9bb9-84c1c22ccb71 /tidb/cdc/default/default/upstream/7362126888831009041 {"id":7362126888831009041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dcde575b-a5de-4713-9bb9-84c1c22ccb71 {"id":"dcde575b-a5de-4713-9bb9-84c1c22ccb71","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b323418 dcde575b-a5de-4713-9bb9-84c1c22ccb71 /tidb/cdc/default/default/upstream/7362126888831009041 {"id":7362126888831009041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x lease 22318f1a0b323418 revoked table capture_session_done_during_task.t exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= check diff successfully WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. check diff failed 1-th time, retry later Collecting requests==2.26.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:56:14 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Installing collected packages: urllib3, idna, certifi, requests Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later check diff failed 5-th time, retry later check diff failed 6-th time, retry later check diff failed 7-th time, retry later table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d30000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31181, start at 2024-04-26 18:56:27.995377661 +0800 CST m=+4.456471338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:27.980 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:27.980 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log check diff failed 8-th time, retry later table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b72c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:27073, start at 2024-04-26 18:56:29.658604325 +0800 CST m=+4.385969849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log table consistent_replicate_storage_file.check1 not exists for 18-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 9-th time, retry later Starting CDC state checker... Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 19-th check, retry later check diff failed 10-th time, retry later table consistent_replicate_storage_file.check1 not exists for 20-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 11-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 12-th time, retry later table consistent_replicate_storage_file.check1 not exists for 21-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683348900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25568, start at 2024-04-26 18:56:38.978504568 +0800 CST m=+4.433707408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:38.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:38.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:38.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later check diff failed 13-th time, retry later table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 14-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check_changefeed_state http://127.0.0.1:2379 cb026ec9-1032-40ef-aa0f-67f605ce3a55 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=cb026ec9-1032-40ef-aa0f-67f605ce3a55 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cb026ec9-1032-40ef-aa0f-67f605ce3a55 -s + info='{ "upstream_id": 7362126685575130260, "namespace": "default", "id": "cb026ec9-1032-40ef-aa0f-67f605ce3a55", "state": "finished", "checkpoint_tso": 449348616826388483, "checkpoint_time": "2024-04-26 18:55:39.920", "error": null }' + echo '{ "upstream_id": 7362126685575130260, "namespace": "default", "id": "cb026ec9-1032-40ef-aa0f-67f605ce3a55", "state": "finished", "checkpoint_tso": 449348616826388483, "checkpoint_time": "2024-04-26 18:55:39.920", "error": null }' { "upstream_id": 7362126685575130260, "namespace": "default", "id": "cb026ec9-1032-40ef-aa0f-67f605ce3a55", "state": "finished", "checkpoint_tso": 449348616826388483, "checkpoint_time": "2024-04-26 18:55:39.920", "error": null } ++ echo '{' '"upstream_id":' 7362126685575130260, '"namespace":' '"default",' '"id":' '"cb026ec9-1032-40ef-aa0f-67f605ce3a55",' '"state":' '"finished",' '"checkpoint_tso":' 449348616826388483, '"checkpoint_time":' '"2024-04-26' '18:55:39.920",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7362126685575130260, '"namespace":' '"default",' '"id":' '"cb026ec9-1032-40ef-aa0f-67f605ce3a55",' '"state":' '"finished",' '"checkpoint_tso":' 449348616826388483, '"checkpoint_time":' '"2024-04-26' '18:55:39.920",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:56:38 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... check diff failed 15-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later check diff failed 16-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later check diff failed 17-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later check diff failed 18-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file.t2 exists table big_txn.usertable1 exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 19-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b72c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:27073, start at 2024-04-26 18:56:29.658604325 +0800 CST m=+4.385969849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 18:56:54 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.2586925871.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1c50001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:1024, start at 2024-04-26 18:54:57.856474904 +0800 CST m=+4.286730751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 18:56:57 CST 2024] <<<<<< run test case clustered_index success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:56:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {ClusterID:default ID:default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 Info:{"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} Status:0xc002c26780 TaskPositions:map[868d067f-c047-4e0d-8021-dff7282b7cf0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 91, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 84, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 87, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 92, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 82, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 88, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 89, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/868d067f-c047-4e0d-8021-dff7282b7cf0 {"id":"868d067f-c047-4e0d-8021-dff7282b7cf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13c07e 868d067f-c047-4e0d-8021-dff7282b7cf0 /tidb/cdc/default/default/changefeed/info/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} /tidb/cdc/default/default/changefeed/status/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":449348621581418516,"min-table-barrier-ts":449348636988932101,"admin-job-type":0} /tidb/cdc/default/default/task/position/868d067f-c047-4e0d-8021-dff7282b7cf0/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {ClusterID:default ID:default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 Info:{"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} Status:0xc002c26780 TaskPositions:map[868d067f-c047-4e0d-8021-dff7282b7cf0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 91, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 84, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 87, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 92, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 82, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 88, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 89, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/868d067f-c047-4e0d-8021-dff7282b7cf0 {"id":"868d067f-c047-4e0d-8021-dff7282b7cf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13c07e 868d067f-c047-4e0d-8021-dff7282b7cf0 /tidb/cdc/default/default/changefeed/info/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} /tidb/cdc/default/default/changefeed/status/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":449348621581418516,"min-table-barrier-ts":449348636988932101,"admin-job-type":0} /tidb/cdc/default/default/task/position/868d067f-c047-4e0d-8021-dff7282b7cf0/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {ClusterID:default ID:default/08bcdf0e-ca96-4b09-b1d7-c7127deee387 Info:{"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} Status:0xc002c26780 TaskPositions:map[868d067f-c047-4e0d-8021-dff7282b7cf0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 91, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 84, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 87, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 92, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 82, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 88, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing tableID: 89, resolvedTs: 449348621581418516, checkpointTs: 449348621581418516, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/868d067f-c047-4e0d-8021-dff7282b7cf0 {"id":"868d067f-c047-4e0d-8021-dff7282b7cf0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0b13c07e 868d067f-c047-4e0d-8021-dff7282b7cf0 /tidb/cdc/default/default/changefeed/info/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"upstream-id":7362126887939402613,"namespace":"default","changefeed-id":"08bcdf0e-ca96-4b09-b1d7-c7127deee387","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:54:57.71827538+08:00","start-ts":449348605741105156,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protoco+ grep -q 'etcd info' l":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348605754212358} /tidb/cdc/default/default/changefeed/status/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":449348621581418516,"min-table-barrier-ts":449348636988932101,"admin-job-type":0} /tidb/cdc/default/default/task/position/868d067f-c047-4e0d-8021-dff7282b7cf0/08bcdf0e-ca96-4b09-b1d7-c7127deee387 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362126887939402613 {"id":7362126887939402613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d30000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31181, start at 2024-04-26 18:56:27.995377661 +0800 CST m=+4.456471338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:27.980 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:27.980 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... Edition: Community TiKV Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 Release Version: 6.5.9 Edition: Community UTC Build Time: 2024-04-25 09:35:48 Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349ce8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26640, start at 2024-04-26 18:57:00.745499395 +0800 CST m=+4.436279032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d45440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:1015, start at 2024-04-26 18:55:00.461086116 +0800 CST m=+4.508163034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.489 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d297c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:1075, start at 2024-04-26 18:54:58.677416525 +0800 CST m=+4.484700001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.655 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.655 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d31d00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:1060, start at 2024-04-26 18:54:59.210999328 +0800 CST m=+6.739480592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d81900019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1105, start at 2024-04-26 18:55:04.341559791 +0800 CST m=+4.549014764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.341 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.341 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683348900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25568, start at 2024-04-26 18:56:38.978504568 +0800 CST m=+4.433707408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:38.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:38.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:38.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d718c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:1006, start at 2024-04-26 18:55:03.288905658 +0800 CST m=+4.444445111 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.267 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.267 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:02.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:02.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d6c100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:1046, start at 2024-04-26 18:55:02.927267993 +0800 CST m=+4.576411032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dce4c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1287, start at 2024-04-26 18:55:09.244620583 +0800 CST m=+4.388236727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b7100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:26976, start at 2024-04-26 18:56:29.664958022 +0800 CST m=+4.443907163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.636 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.636 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6832b72c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:27073, start at 2024-04-26 18:56:29.658604325 +0800 CST m=+4.385969849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:29.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:29.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:29.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:14 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3143731439.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:946, start at 2024-04-26 18:54:57.742737154 +0800 CST m=+4.256121956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d1c50001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:1024, start at 2024-04-26 18:54:57.856474904 +0800 CST m=+4.286730751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:57.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:57.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:57.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:16 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sinkv2/eventsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2083720839.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:17 GMT < Content-Length: 656 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 31442 owner id 5de8aae8-ccc9-4700-81a6-e97e761d87d6 default [Fri Apr 26 18:57:17 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3156731569.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e9c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:1078, start at 2024-04-26 18:55:05.149214429 +0800 CST m=+4.336252621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:19 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.20895.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 1a546746-4b79-4ea6-bcb5-760ef8cd06d8 Info: {"upstream_id":7362127521033598397,"namespace":"default","id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:19.570112336+08:00","start_ts":449348642915745797,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348642915745797,"checkpoint_ts":449348642915745797,"checkpoint_time":"2024-04-26 18:57:19.443"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31097, start at 2024-04-26 18:56:28.022754913 +0800 CST m=+4.542218322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:28.034 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:28.034 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68329d30000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:31181, start at 2024-04-26 18:56:27.995377661 +0800 CST m=+4.456471338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:28.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:27.980 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:27.980 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Fri Apr 26 18:57:19 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3515835160.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:20 GMT < Content-Length: 909 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5de8aae8-ccc9-4700-81a6-e97e761d87d6 {"id":"5de8aae8-ccc9-4700-81a6-e97e761d87d6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b2f 5de8aae8-ccc9-4700-81a6-e97e761d87d6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "5de8aae8-ccc9-4700-81a6-e97e761d87d6 default" | grep id Starting Upstream TiDB... "id": "aaad778c-05bf-4806-911b-b7337ae3db27", run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: aaad778c-05bf-4806-911b-b7337ae3db27 pass test: check health pass test: get status VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d46380019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:937, start at 2024-04-26 18:55:00.542196083 +0800 CST m=+4.667130928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d45440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:1015, start at 2024-04-26 18:55:00.461086116 +0800 CST m=+4.508163034 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:00.489 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:00.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:00.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.18665.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: create changefeed + set +x + tso='449348643785277441 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348643785277441 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:57:23 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.1868418686.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:22 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/414bd64b-b007-4c43-923d-974cc9f0edee {"id":"414bd64b-b007-4c43-923d-974cc9f0edee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d43ad91 414bd64b-b007-4c43-923d-974cc9f0edee /tidb/cdc/default/default/upstream/7362127913796743276 {"id":7362127913796743276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/414bd64b-b007-4c43-923d-974cc9f0edee {"id":"414bd64b-b007-4c43-923d-974cc9f0edee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d43ad91 414bd64b-b007-4c43-923d-974cc9f0edee /tidb/cdc/default/default/upstream/7362127913796743276 {"id":7362127913796743276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/414bd64b-b007-4c43-923d-974cc9f0edee {"id":"414bd64b-b007-4c43-923d-974cc9f0edee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d43ad91 414bd64b-b007-4c43-923d-974cc9f0edee /tidb/cdc/default/default/upstream/7362127913796743276 {"id":7362127913796743276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 1be74671-0c42-4813-8e89-21c57961a380 Info: {"upstream_id":7362127913796743276,"namespace":"default","id":"1be74671-0c42-4813-8e89-21c57961a380","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:23.089150569+08:00","start_ts":449348643016146949,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348643016146949,"checkpoint_ts":449348643016146949,"checkpoint_time":"2024-04-26 18:57:19.826"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d7e780019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1021, start at 2024-04-26 18:55:04.118739773 +0800 CST m=+4.383554252 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.094 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.094 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d81900019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:1105, start at 2024-04-26 18:55:04.341559791 +0800 CST m=+4.549014764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:04.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:04.341 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:04.341 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:23 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.2089120893.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835dbdc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25794, start at 2024-04-26 18:57:21.179435222 +0800 CST m=+4.458657007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349ce8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26640, start at 2024-04-26 18:57:00.745499395 +0800 CST m=+4.436279032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d27140019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:981, start at 2024-04-26 18:54:58.543900384 +0800 CST m=+4.446907387 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.575 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d297c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:1075, start at 2024-04-26 18:54:58.677416525 +0800 CST m=+4.484700001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:58.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:58.655 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:58.655 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:24 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.2266022662.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_redo_resolved_ts 08bcdf0e-ca96-4b09-b1d7-c7127deee387 449348643568484358 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/08bcdf0e-ca96-4b09-b1d7-c7127deee387/meta global resolved ts 449348637277028357 not forward to 449348643568484358 run task failed 1-th time, retry later check_redo_resolved_ts 08bcdf0e-ca96-4b09-b1d7-c7127deee387 449348643568484358 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/08bcdf0e-ca96-4b09-b1d7-c7127deee387/meta global resolved ts 449348643280125956 not forward to 449348643568484358 run task failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d30d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:973, start at 2024-04-26 18:54:59.147749812 +0800 CST m=+6.795821752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.124 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.124 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d31d00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:1060, start at 2024-04-26 18:54:59.210999328 +0800 CST m=+6.739480592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:57:59.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:54:59.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:44:59.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:24 CST 2024] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.2504525047.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:26 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a46d1e38-fecf-4f45-a35a-edd0b3240ec2 {"id":"a46d1e38-fecf-4f45-a35a-edd0b3240ec2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d578a90 a46d1e38-fecf-4f45-a35a-edd0b3240ec2 /tidb/cdc/default/default/upstream/7362127535385101024 {"id":7362127535385101024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a46d1e38-fecf-4f45-a35a-edd0b3240ec2 {"id":"a46d1e38-fecf-4f45-a35a-edd0b3240ec2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d578a90 a46d1e38-fecf-4f45-a35a-edd0b3240ec2 /tidb/cdc/default/default/upstream/7362127535385101024 {"id":7362127535385101024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a46d1e38-fecf-4f45-a35a-edd0b3240ec2 {"id":"a46d1e38-fecf-4f45-a35a-edd0b3240ec2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d578a90 a46d1e38-fecf-4f45-a35a-edd0b3240ec2 /tidb/cdc/default/default/upstream/7362127535385101024 {"id":7362127535385101024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.18742.out cli changefeed create --start-ts=449348643785277441 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: b10b7a2e-c4a0-461c-a232-9be02350c790 Info: {"upstream_id":7362127535385101024,"namespace":"default","id":"b10b7a2e-c4a0-461c-a232-9be02350c790","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-04-26T18:57:26.491729202+08:00","start_ts":449348643785277441,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348643785277441,"checkpoint_ts":449348643785277441,"checkpoint_time":"2024-04-26 18:57:22.760"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:26 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd09e818-6497-4bfb-99fc-204a52d08810 {"id":"dd09e818-6497-4bfb-99fc-204a52d08810","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5ded8e dd09e818-6497-4bfb-99fc-204a52d08810 /tidb/cdc/default/default/upstream/7362127552999933844 {"id":7362127552999933844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd09e818-6497-4bfb-99fc-204a52d08810 {"id":"dd09e818-6497-4bfb-99fc-204a52d08810","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5ded8e dd09e818-6497-4bfb-99fc-204a52d08810 /tidb/cdc/default/default/upstream/7362127552999933844 {"id":7362127552999933844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd09e818-6497-4bfb-99fc-204a52d08810 {"id":"dd09e818-6497-4bfb-99fc-204a52d08810","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5ded8e dd09e818-6497-4bfb-99fc-204a52d08810 /tidb/cdc/default/default/upstream/7362127552999933844 {"id":7362127552999933844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.20950.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: 29b72ef5-8f58-47bc-816a-3c39d66c3b5b Info: {"upstream_id":7362127552999933844,"namespace":"default","id":"29b72ef5-8f58-47bc-816a-3c39d66c3b5b","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-04-26T18:57:26.984389748+08:00","start_ts":449348644869242884,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348644869242884,"checkpoint_ts":449348644869242884,"checkpoint_time":"2024-04-26 18:57:26.895"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:27 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f8bc23d-4c7a-4356-915d-daaf08b3b679 {"id":"0f8bc23d-4c7a-4356-915d-daaf08b3b679","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5a5990 0f8bc23d-4c7a-4356-915d-daaf08b3b679 /tidb/cdc/default/default/upstream/7362127524722733083 {"id":7362127524722733083,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f8bc23d-4c7a-4356-915d-daaf08b3b679 {"id":"0f8bc23d-4c7a-4356-915d-daaf08b3b679","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5a5990 0f8bc23d-4c7a-4356-915d-daaf08b3b679 /tidb/cdc/default/default/upstream/7362127524722733083 {"id":7362127524722733083,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f8bc23d-4c7a-4356-915d-daaf08b3b679 {"id":"0f8bc23d-4c7a-4356-915d-daaf08b3b679","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5a5990 0f8bc23d-4c7a-4356-915d-daaf08b3b679 /tidb/cdc/default/default/upstream/7362127524722733083 {"id":7362127524722733083,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68334afc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25485, start at 2024-04-26 18:56:39.103667615 +0800 CST m=+4.635696829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:39.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:39.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:39.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683348900015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:25568, start at 2024-04-26 18:56:38.978504568 +0800 CST m=+4.433707408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:38.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:56:38.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:46:38.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:27 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2957429576.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:27 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e352fb-5ad7-412d-a66a-0ea61542849d {"id":"45e352fb-5ad7-412d-a66a-0ea61542849d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5af48e 45e352fb-5ad7-412d-a66a-0ea61542849d /tidb/cdc/default/default/upstream/7362127521332938847 {"id":7362127521332938847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e352fb-5ad7-412d-a66a-0ea61542849d {"id":"45e352fb-5ad7-412d-a66a-0ea61542849d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5af48e 45e352fb-5ad7-412d-a66a-0ea61542849d /tidb/cdc/default/default/upstream/7362127521332938847 {"id":7362127521332938847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e352fb-5ad7-412d-a66a-0ea61542849d {"id":"45e352fb-5ad7-412d-a66a-0ea61542849d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d5af48e 45e352fb-5ad7-412d-a66a-0ea61542849d /tidb/cdc/default/default/upstream/7362127521332938847 {"id":7362127521332938847,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.25109.out cli changefeed create --start-ts=449348644328701953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: 2afe3aba-1500-44bc-b04e-306bd0751854 Info: {"upstream_id":7362127521332938847,"namespace":"default","id":"2afe3aba-1500-44bc-b04e-306bd0751854","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:28.281803597+08:00","start_ts":449348644328701953,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348644328701953,"checkpoint_ts":449348644328701953,"checkpoint_time":"2024-04-26 18:57:24.833"} PASS start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x check_redo_resolved_ts 08bcdf0e-ca96-4b09-b1d7-c7127deee387 449348643568484358 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/08bcdf0e-ca96-4b09-b1d7-c7127deee387/meta run task successfully Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/04/26 18:57:29.182 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 18:57:29.183 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 18:57:29.183 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348637460529156] [resolvedTs=449348644361732102] [2024/04/26 18:57:29.188 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 18:57:29.188 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 18:57:29.189 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=6.046732ms] [2024/04/26 18:57:29.197 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 18:57:29.200 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.202 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.203 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.203 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 18:57:29.203 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 18:57:29.203 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 18:57:29.203 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 18:57:29.209 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 18:57:29.212 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 18:57:29.745 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=561.70564ms] [2024/04/26 18:57:29.745 +08:00] [ERROR] [redo.go:265] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449348637447421956,\"CommitTs\":449348637460529156,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_file\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d70980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:931, start at 2024-04-26 18:55:03.2122498 +0800 CST m=+4.455244539 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d718c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:1006, start at 2024-04-26 18:55:03.288905658 +0800 CST m=+4.444445111 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:03.267 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:03.267 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.25390.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 18:57:30.781 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348644361732102] [2024/04/26 18:57:30.781 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348644361732102] [2024/04/26 18:57:30.882 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348644361732102] [2024/04/26 18:57:30.882 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348644361732102] + set +x + tso='449348645809029121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348645809029121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:57:30 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.2543225434.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:02.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d69a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:962, start at 2024-04-26 18:55:02.797183296 +0800 CST m=+4.532424474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:02.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d6c100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:1046, start at 2024-04-26 18:55:02.927267993 +0800 CST m=+4.576411032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:03.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:02.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:02.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.18443.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 18:57:31.384 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=449348644361732102] [2024/04/26 18:57:31.384 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=449348644361732102] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:31 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 18:57:31 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.2964229644.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449348645982568449 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348645982568449 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:57:31 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.1847318475.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:57:31.484 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449348644361732102] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 18:57:31.484 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] Apply redo log successfully Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dcfdc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1207, start at 2024-04-26 18:55:09.343544553 +0800 CST m=+4.556782284 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.372 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682dce4c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:1287, start at 2024-04-26 18:55:09.244620583 +0800 CST m=+4.388236727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:09.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:09.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:09.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff successfully [Fri Apr 26 18:57:32 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:32 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2396323965.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:33 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96236cfb-09b3-46a7-afce-95eeb8f5f942 {"id":"96236cfb-09b3-46a7-afce-95eeb8f5f942","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d6fd48d 96236cfb-09b3-46a7-afce-95eeb8f5f942 /tidb/cdc/default/default/upstream/7362127537732048636 {"id":7362127537732048636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96236cfb-09b3-46a7-afce-95eeb8f5f942 {"id":"96236cfb-09b3-46a7-afce-95eeb8f5f942","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d6fd48d 96236cfb-09b3-46a7-afce-95eeb8f5f942 /tidb/cdc/default/default/upstream/7362127537732048636 {"id":7362127537732048636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96236cfb-09b3-46a7-afce-95eeb8f5f942 {"id":"96236cfb-09b3-46a7-afce-95eeb8f5f942","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d6fd48d 96236cfb-09b3-46a7-afce-95eeb8f5f942 /tidb/cdc/default/default/upstream/7362127537732048636 {"id":7362127537732048636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.25494.out cli changefeed create --start-ts=449348645809029121 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Create changefeed successfully! ID: cc360117-92cb-4487-8241-f2320924084f Info: {"upstream_id":7362127537732048636,"namespace":"default","id":"cc360117-92cb-4487-8241-f2320924084f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:34.347142522+08:00","start_ts":449348645809029121,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348645809029121,"checkpoint_ts":449348645809029121,"checkpoint_time":"2024-04-26 18:57:30.480"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:34 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53836a61-c49e-4b56-8520-5860955528b4 {"id":"53836a61-c49e-4b56-8520-5860955528b4","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d0b2 53836a61-c49e-4b56-8520-5860955528b4 /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53836a61-c49e-4b56-8520-5860955528b4 {"id":"53836a61-c49e-4b56-8520-5860955528b4","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d0b2 53836a61-c49e-4b56-8520-5860955528b4 /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53836a61-c49e-4b56-8520-5860955528b4 {"id":"53836a61-c49e-4b56-8520-5860955528b4","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/71ade31f-5912-4d69-b819-9069ee52b5fc {"id":"71ade31f-5912-4d69-b819-9069ee52b5fc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d090 71ade31f-5912-4d69-b819-9069ee52b5fc /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d67d0b2 53836a61-c49e-4b56-8520-5860955528b4 /tidb/cdc/default/default/upstream/7362127955994482530 {"id":7362127955994482530,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:34 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 {"id":"f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d729b90 f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 /tidb/cdc/default/default/upstream/7362127545469651612 {"id":7362127545469651612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 {"id":"f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d729b90 f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 /tidb/cdc/default/default/upstream/7362127545469651612 {"id":7362127545469651612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 {"id":"f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d729b90 f1cfe04b-7df8-4809-b9e2-3aa2bf8a51f6 /tidb/cdc/default/default/upstream/7362127545469651612 {"id":7362127545469651612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.18537.out cli changefeed create --start-ts=449348645982568449 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 16904e33-5d2a-4eda-9c64-c7b1b8a8de56 Info: {"upstream_id":7362127545469651612,"namespace":"default","id":"16904e33-5d2a-4eda-9c64-c7b1b8a8de56","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:35.031837134+08:00","start_ts":449348645982568449,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348645982568449,"checkpoint_ts":449348645982568449,"checkpoint_time":"2024-04-26 18:57:31.142"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:35 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa80127-38ac-4992-b35e-55c341802729 {"id":"9aa80127-38ac-4992-b35e-55c341802729","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7a8d 9aa80127-38ac-4992-b35e-55c341802729 /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa80127-38ac-4992-b35e-55c341802729 {"id":"9aa80127-38ac-4992-b35e-55c341802729","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7a8d 9aa80127-38ac-4992-b35e-55c341802729 /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9aa80127-38ac-4992-b35e-55c341802729 {"id":"9aa80127-38ac-4992-b35e-55c341802729","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7a8d 9aa80127-38ac-4992-b35e-55c341802729 /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info: {"upstream_id":7362127572630951477,"namespace":"default","id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T18:57:35.498901544+08:00","start_ts":449348647108739073,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348647108739073,"checkpoint_ts":449348647108739073,"checkpoint_time":"2024-04-26 18:57:35.438"} coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:985, start at 2024-04-26 18:55:05.146143522 +0800 CST m=+4.417380626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c682d8e9c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:1078, start at 2024-04-26 18:55:05.149214429 +0800 CST m=+4.336252621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:58:05.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:55:05.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:45:05.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.25187.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348647709310977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348647709310977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:57:38 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.2522025222.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:57:39.448 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00202c1c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:41 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f39f55bc-b7c9-42b3-a591-96286bc4746b {"id":"f39f55bc-b7c9-42b3-a591-96286bc4746b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d900f90 f39f55bc-b7c9-42b3-a591-96286bc4746b /tidb/cdc/default/default/upstream/7362127557280529974 {"id":7362127557280529974,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f39f55bc-b7c9-42b3-a591-96286bc4746b {"id":"f39f55bc-b7c9-42b3-a591-96286bc4746b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d900f90 f39f55bc-b7c9-42b3-a591-96286bc4746b /tidb/cdc/default/default/upstream/7362127557280529974 {"id":7362127557280529974,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f39f55bc-b7c9-42b3-a591-96286bc4746b {"id":"f39f55bc-b7c9-42b3-a591-96286bc4746b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d900f90 f39f55bc-b7c9-42b3-a591-96286bc4746b /tidb/cdc/default/default/upstream/7362127557280529974 {"id":7362127557280529974,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.25282.out cli changefeed create --start-ts=449348647709310977 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 3cd5680f-f6f4-4b7c-bf9d-7547fa9bdca9 Info: {"upstream_id":7362127557280529974,"namespace":"default","id":"3cd5680f-f6f4-4b7c-bf9d-7547fa9bdca9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:57:41.457429918+08:00","start_ts":449348647709310977,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348647709310977,"checkpoint_ts":449348647709310977,"checkpoint_time":"2024-04-26 18:57:37.729"} PASS Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349b580019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26538, start at 2024-04-26 18:57:00.657236683 +0800 CST m=+4.398961756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68349ce8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:26640, start at 2024-04-26 18:57:00.745499395 +0800 CST m=+4.436279032 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:00.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:00.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:00.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:57:43 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.3069230694.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:57:46 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ff88a6-5948-4d86-b35b-16702a0a1a9a {"id":"22ff88a6-5948-4d86-b35b-16702a0a1a9a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0da5c48e 22ff88a6-5948-4d86-b35b-16702a0a1a9a /tidb/cdc/default/default/upstream/7362128051850398086 {"id":7362128051850398086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ff88a6-5948-4d86-b35b-16702a0a1a9a {"id":"22ff88a6-5948-4d86-b35b-16702a0a1a9a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0da5c48e 22ff88a6-5948-4d86-b35b-16702a0a1a9a /tidb/cdc/default/default/upstream/7362128051850398086 {"id":7362128051850398086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ff88a6-5948-4d86-b35b-16702a0a1a9a {"id":"22ff88a6-5948-4d86-b35b-16702a0a1a9a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0da5c48e 22ff88a6-5948-4d86-b35b-16702a0a1a9a /tidb/cdc/default/default/upstream/7362128051850398086 {"id":7362128051850398086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835dbdc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25794, start at 2024-04-26 18:57:21.179435222 +0800 CST m=+4.458657007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382ef40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27280, start at 2024-04-26 18:57:59.256776107 +0800 CST m=+8.158011081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_resolved_ts_fallback.t1 exists [Fri Apr 26 18:58:17 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.2104821050.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.simple1 not exists for 1-th check, retry later table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (31442) - No such process wait process 31442 exit for 1-th time... process 31442 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep aaad778c-05bf-4806-911b-b7337ae3db27 -A1 | grep '"is-owner": true' "is-owner": true, run task successfully [Fri Apr 26 18:58:18 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.3190031902.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835da800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25704, start at 2024-04-26 18:57:21.075376544 +0800 CST m=+4.443562993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6835dbdc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:25794, start at 2024-04-26 18:57:21.179435222 +0800 CST m=+4.458657007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:21.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:21.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:21.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.29689.out cli tso query --pd=http://127.0.0.1:2379 table autorandom_test.table_a not exists for 1-th check, retry later table autorandom_test.table_a exists check diff failed 1-th time, retry later + set +x + tso='449348658530615298 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348658530615298 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 18:58:19 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.2973129733.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {ClusterID:default ID:default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 Info:{"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} Status:0xc00138bce0 TaskPositions:map[17a0caa1-aaa8-4417-97d6-c24890f7a73c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ed96d486-6add-481c-b6a8-273f8bfa3387:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a0caa1-aaa8-4417-97d6-c24890f7a73c {"id":"17a0caa1-aaa8-4417-97d6-c24890f7a73c","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5cac 17a0caa1-aaa8-4417-97d6-c24890f7a73c /tidb/cdc/default/default/changefeed/info/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} /tidb/cdc/default/default/changefeed/status/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":449348658604802057,"min-table-barrier-ts":449348658604802057,"admin-job-type":0} /tidb/cdc/default/default/task/position/17a0caa1-aaa8-4417-97d6-c24890f7a73c/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ed96d486-6add-481c-b6a8-273f8bfa3387/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {ClusterID:default ID:default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 Info:{"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} Status:0xc00138bce0 TaskPositions:map[17a0caa1-aaa8-4417-97d6-c24890f7a73c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ed96d486-6add-481c-b6a8-273f8bfa3387:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a0caa1-aaa8-4417-97d6-c24890f7a73c {"id":"17a0caa1-aaa8-4417-97d6-c24890f7a73c","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5cac 17a0caa1-aaa8-4417-97d6-c24890f7a73c /tidb/cdc/default/default/changefeed/info/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} /tidb/cdc/default/default/changefeed/status/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":449348658604802057,"min-table-barrier-ts":449348658604802057,"admin-job-type":0} /tidb/cdc/default/default/task/position/17a0caa1-aaa8-4417-97d6-c24890f7a73c/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ed96d486-6add-481c-b6a8-273f8bfa3387/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {ClusterID:default ID:default/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 Info:{"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} Status:0xc00138bce0 TaskPositions:map[17a0caa1-aaa8-4417-97d6-c24890f7a73c:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ed96d486-6add-481c-b6a8-273f8bfa3387:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a0caa1-aaa8-4417-97d6-c24890f7a73c {"id":"17a0caa1-aaa8-4417-97d6-c24890f7a73c","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ed96d486-6add-481c-b6a8-273f8bfa3387 {"id":"ed96d486-6add-481c-b6a8-273f8bfa3387","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5b8e ed96d486-6add-481c-b6a8-273f8bfa3387 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d3b5cac 17a0caa1-aaa8-4417-97d6-c24890f7a73c /tidb/cdc/default/default/changefeed/info/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"upstream-id":7362127521033598397,"namespace":"default","changefeed-id":"1a546746-4b79-4ea6-bcb5-760ef8cd06d8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:19.570112336+08:00","start-ts":449348642915745797,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348642941960197} /tidb/cdc/default/default/changefeed/status/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":449348658604802057,"min-table-barrier-ts":449348658604802057,"admin-job-type":0} /tidb/cdc/default/default/task/position/17a0caa1-aaa8-4417-97d6-c24890f7a73c/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ed96d486-6add-481c-b6a8-273f8bfa3387/1a546746-4b79-4ea6-bcb5-760ef8cd06d8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127521033598397 {"id":7362127521033598397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:21 CST 2024] <<<<<< run test case autorandom success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} Status:0xc000f52ff0 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} Status:0xc000f53128 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3d11f8-9e50-489c-ae27-56cfd24eeed1 {"id":"5e3d11f8-9e50-489c-ae27-56cfd24eeed1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334c0e 5e3d11f8-9e50-489c-ae27-56cfd24eeed1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.339919483+08:00","start-ts":449348643645292548,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348658063212552} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449348656411443225,"min-table-barrier-ts":449348656555622402,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} Status:0xc000f52ff0 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} Status:0xc000f53128 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3d11f8-9e50-489c-ae27-56cfd24eeed1 {"id":"5e3d11f8-9e50-489c-ae27-56cfd24eeed1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334c0e 5e3d11f8-9e50-489c-ae27-56cfd24eeed1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.339919483+08:00","start-ts":449348643645292548,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348658063212552} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449348656411443225,"min-table-barrier-ts":449348656555622402,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {ClusterID:default ID:default/changefeed-test1 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} Status:0xc000f52ff0 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} changefeedID: default/changefeed-test3 {ClusterID:default ID:default/changefeed-test3 Info:{"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole:","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} Status:0xc000f53128 TaskPositions:map[5e3d11f8-9e50-489c-ae27-56cfd24eeed1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} aaad778c-05bf-4806-911b-b7337ae3db27:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e3d11f8-9e50-489c-ae27-56cfd24eeed1 {"id":"5e3d11f8-9e50-489c-ae27-56cfd24eeed1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/aaad778c-05bf-4806-911b-b7337ae3db27 {"id":"aaad778c-05bf-4806-911b-b7337ae3db27","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334b4c aaad778c-05bf-4806-911b-b7337ae3db27 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d334c0e 5e3d11f8-9e50-489c-ae27-56cfd24eeed1 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:57:22.13552136+08:00","start-ts":449348643605970949,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643618816003} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.339919483+08:00","start-ts":449348643645292548,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348658063212552} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7362127918734367578,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-04-26T18:57:22.477603861+08:00","start-ts":449348643697459203,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348643710828546} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449348656411443225,"min-table-barrier-ts":449348656555622402,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449348656555622402,"min-table-barrier-ts":449348656555622402,"admin-job-type":0} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5e3d11f8-9e50-489c-ae27-56cfd24eeed1/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/aaad778c-05bf-4806-911b-b7337ae3db27/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127918734367578 {"id":7362127918734367578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' "address": "127.0.0.1:8300", "address": "127.0.0.1:8300", run task successfully pass test: list changefeed Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore pass test: get changefeed Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:22 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0ac7d44-45c4-4447-9188-3902d78e1743 {"id":"e0ac7d44-45c4-4447-9188-3902d78e1743","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e2f7190 e0ac7d44-45c4-4447-9188-3902d78e1743 /tidb/cdc/default/default/upstream/7362128140908178141 {"id":7362128140908178141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0ac7d44-45c4-4447-9188-3902d78e1743 {"id":"e0ac7d44-45c4-4447-9188-3902d78e1743","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e2f7190 e0ac7d44-45c4-4447-9188-3902d78e1743 /tidb/cdc/default/default/upstream/7362128140908178141 {"id":7362128140908178141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0ac7d44-45c4-4447-9188-3902d78e1743 {"id":"e0ac7d44-45c4-4447-9188-3902d78e1743","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e2f7190 e0ac7d44-45c4-4447-9188-3902d78e1743 /tidb/cdc/default/default/upstream/7362128140908178141 {"id":7362128140908178141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.29794.out cli changefeed create --start-ts=449348658530615298 --sink-uri=mysql://root@127.0.0.1:3306/ pass test: resume changefeed Create changefeed successfully! ID: 2dfb8201-67c7-433c-86c4-105885c4ed16 Info: {"upstream_id":7362128140908178141,"namespace":"default","id":"2dfb8201-67c7-433c-86c4-105885c4ed16","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-04-26T18:58:23.11028735+08:00","start_ts":449348658530615298,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348658530615298,"checkpoint_ts":449348658530615298,"checkpoint_time":"2024-04-26 18:58:19.009"} PASS table processor_resolved_ts_fallback.t3 not exists for 2-th check, retry later pass test: pause changefeed pass test: rebalance table table owner_remove_table_error.finished_mark not exists for 1-th check, retry later table owner_remove_table_error.finished_mark not exists for 2-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x pass test: move table pass test: get processors VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382ef40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27280, start at 2024-04-26 18:57:59.256776107 +0800 CST m=+8.158011081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... pass test: list processors table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' pass test: set log level 2 run task successfully killing cdc1: 20842 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' table owner_remove_table_error.finished_mark not exists for 3-th check, retry later Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later pass test: remove changefeed pass test: resign owner check diff successfully ERROR 1146 (42S02) at line 1: Table 'sequence_test.seq0' doesn't exist table owner_remove_table_error.finished_mark exists cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused check diff successfully run task failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:27 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> table sequence_test.mark_table not exists for 1-th check, retry later [2024/04/26 18:58:28.859 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table kill_owner_with_ddl.t1 not exists for 1-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v1/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later table test.finish_mark not exists for 1-th check, retry later table sequence_test.mark_table exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:32 CST 2024] <<<<<< run test case sequence success! >>>>>> table kill_owner_with_ddl.t1 exists table test.finish_mark not exists for 2-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (23968) - No such process wait process 23968 exit for 1-th time... process 23968 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 18:58:33 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2417724179.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [mysql] 2024/04/26 18:58:33 connection.go:299: invalid connection table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table test.finish_mark not exists for 3-th check, retry later table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t3 not exists for 2-th check, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t4 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc001117fc8 TaskPositions:map[292229f0-a0bc-4f0f-ae37-efff69ea75fe:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663086153732, checkpointTs: 449348663086153731, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292229f0-a0bc-4f0f-ae37-efff69ea75fe {"id":"292229f0-a0bc-4f0f-ae37-efff69ea75fe","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7b9a 292229f0-a0bc-4f0f-ae37-efff69ea75fe /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663086153731,"min-table-barrier-ts":449348663086153731,"admin-job-type":0} /tidb/cdc/default/default/task/position/292229f0-a0bc-4f0f-ae37-efff69ea75fe/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc001117fc8 TaskPositions:map[292229f0-a0bc-4f0f-ae37-efff69ea75fe:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663086153732, checkpointTs: 449348663086153731, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292229f0-a0bc-4f0f-ae37-efff69ea75fe {"id":"292229f0-a0bc-4f0f-ae37-efff69ea75fe","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7b9a 292229f0-a0bc-4f0f-ae37-efff69ea75fe /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663086153731,"min-table-barrier-ts":449348663086153731,"admin-job-type":0} /tidb/cdc/default/default/task/position/292229f0-a0bc-4f0f-ae37-efff69ea75fe/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc001117fc8 TaskPositions:map[292229f0-a0bc-4f0f-ae37-efff69ea75fe:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663086153732, checkpointTs: 449348663086153731, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292229f0-a0bc-4f0f-ae37-efff69ea75fe {"id":"292229f0-a0bc-4f0f-ae37-efff69ea75fe","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7b9a 292229f0-a0bc-4f0f-ae37-efff69ea75fe /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663086153731,"min-table-barrier-ts":449348663086153731,"admin-job-type":0} /tidb/cdc/default/default/task/position/292229f0-a0bc-4f0f-ae37-efff69ea75fe/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table test.finish_mark not exists for 4-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/04/26 18:58:34.451 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/04/26 18:58:34.451 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/04/26 18:58:34.551 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs55ab3b18_44a4_4bfb_8c53_1a3a8c70dede"] [2024/04/26 18:58:34.655 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6cc1db8c_5ead_4b79_9889_9da7778d2ed8"] [2024/04/26 18:58:34.753 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs80c8ba85_6e73_4cda_8be0_eab3f838dd1c"] [2024/04/26 18:58:34.908 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf6bfb659_568d_4cb7_ac6e_b2feaef4c46f"] [2024/04/26 18:58:35.009 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs34118a64_921b_4a2a_b294_367bea1c27d5"] [2024/04/26 18:58:35.095 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:35.096 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:35.099 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsddfe8b12_b96e_464a_89a4_121b2c62de8c"] [2024/04/26 18:58:35.125 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 18:58:35.126 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 18:58:35.328 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:35.404 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:35.513 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 18:58:35.514 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 18:58:35.516 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:35.598 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs477353e2_da31_4d80_82bd_f5f3c388a1ec"] [2024/04/26 18:58:35.617 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:35.842 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:35.902 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:36.003 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:36.006 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:36.015 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:36.020 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:36.101 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:36.103 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs85de8036_f3f8_4f05_a3cf_a2509b91733a"] [2024/04/26 18:58:36.199 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 18:58:36.205 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 18:58:36.210 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:36.329 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:36.418 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:36.532 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:36.701 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:36.707 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:36.727 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:36.917 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:36.919 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:37.006 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/04/26 18:58:37.009 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/04/26 18:58:37.011 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 18:58:37.014 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 18:58:37.018 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:37.101 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:37.105 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:37.107 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:37.119 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:37.120 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:37.121 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:37.217 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 18:58:37.222 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 18:58:37.296 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:37.425 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:37.433 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:37.529 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:37.730 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:37.731 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:37.814 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:37.927 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:38.016 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:38.102 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:38.121 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:38.195 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:58:38.206 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:38.209 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:38.213 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:38.228 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 18:58:38.295 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:38.298 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 18:58:38.419 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later [2024/04/26 18:58:38.517 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:58:38.522 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 18:58:38.530 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 18:58:38.626 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:38.699 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:58:38.997 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:39.004 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:39.004 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:39.113 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 139 100 35 100 104 1195 3551 --:--:-- --:--:-- --:--:-- 3586 { "status": true, "message": "" [2024/04/26 18:58:39.297 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:58:39.303 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:39.326 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:39.398 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:39.403 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:58:39.417 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:39.429 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:39.500 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:39.500 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/04/26 18:58:39.510 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] table ddl_attributes.finish_mark not exists for 3-th check, retry later [2024/04/26 18:58:39.615 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:39.627 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table test.finish_mark not exists for 5-th check, retry later [2024/04/26 18:58:39.805 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:39.900 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:40.024 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:58:40.106 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:40.115 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:40.197 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:40.199 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 18:58:40.202 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 18:58:40.205 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/04/26 18:58:40.208 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] check diff successfully [2024/04/26 18:58:40.306 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:40.404 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:40.412 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:40.414 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:40.498 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:58:40.503 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:40.511 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 18:58:40.521 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 18:58:40.630 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:40 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> [2024/04/26 18:58:40.618 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:58:40.622 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:40.731 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:40.801 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:40.834 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:41.019 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] }lease 22318f1a0d67d0b2 revoked check diff failed 1-th time, retry later [2024/04/26 18:58:41.109 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:41.239 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:58:41.437 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 18:58:41.439 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 18:58:41.315 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:41.324 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:41.423 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:41.518 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:41.524 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:41.599 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 18:58:41.600 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 18:58:41.602 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 18:58:41.604 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 18:58:41.767 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 18:58:41.772 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 18:58:41.606 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:58:41.620 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 18:58:41.700 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:41.701 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 18:58:41.704 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:41.802 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:58:41.815 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 18:58:41.817 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.finish_mark not exists for 6-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later [2024/04/26 18:58:41.821 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 18:58:41.896 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:41.919 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:41.924 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:42.117 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:42.198 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:42.320 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:42.132 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 18:58:42.134 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 18:58:42.220 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 18:58:42.222 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 18:58:42.396 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:42.401 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:42.511 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:42.527 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:42.610 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:42.615 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:58:42.581 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 18:58:42.590 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382e380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27181, start at 2024-04-26 18:57:59.2196347 +0800 CST m=+8.176601327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68382ef40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:27280, start at 2024-04-26 18:57:59.256776107 +0800 CST m=+8.158011081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-18:59:59.263 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:57:59.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:47:59.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:58:42 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.3134031342.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:58:42.704 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/04/26 18:58:42.719 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:42.721 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/04/26 18:58:42.724 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:42.836 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 18:58:42.838 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 18:58:42.847 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 18:58:42.848 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later [2024/04/26 18:58:43.002 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:58:43.018 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:43.023 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:43.099 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:43.099 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:43.002 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 18:58:43.010 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 18:58:43.225 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:43.184 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 18:58:43.186 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 18:58:43.251 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 18:58:43.253 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 18:58:43.349 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] table changefeed_pause_resume.t1 not exists for 1-th check, retry later [2024/04/26 18:58:43.413 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:43.434 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 18:58:43.497 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 18:58:43.504 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/04/26 18:58:43.504 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:43.512 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/04/26 18:58:43.619 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:43.621 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:43.406 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 18:58:43.512 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 18:58:43.514 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 18:58:43.578 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 18:58:43.579 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] table test.finish_mark not exists for 7-th check, retry later [2024/04/26 18:58:43.703 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:43.717 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:43.802 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:43.803 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:58:43.831 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 18:58:43.684 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] table ddl_attributes.finish_mark exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:58:43.899 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 18:58:43.999 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:44.014 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:44.114 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:58:44.042 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 18:58:44.134 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 18:58:44.209 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:44.210 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:44.224 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:44.296 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:44.312 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:44.223 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 18:58:44.293 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 18:58:44.295 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 18:58:44.393 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:44 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Starting Upstream TiDB... [2024/04/26 18:58:44.535 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:44.613 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:44.569 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:44.809 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:44.818 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:44.666 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 18:58:44.672 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 18:58:44.748 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 18:58:44.750 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 18:58:44.847 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 18:58:44.910 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 18:58:44.909 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:44.923 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:58:45.022 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:58:45.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:45.110 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/04/26 18:58:45.114 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/04/26 18:58:45.118 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/04/26 18:58:45.125 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/04/26 18:58:45.115 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 18:58:45.126 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24182) - No such process wait process 24182 exit for 1-th time... process 24182 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 18:58:43 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2439624398.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:58:45.212 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:45.220 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:45.313 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:58:45.404 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:45.415 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:45.274 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 18:58:45.279 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table test.finish_mark not exists for 8-th check, retry later check diff failed 3-th time, retry later [2024/04/26 18:58:45.502 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:45.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:58:45.517 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Starting Upstream TiDB... [2024/04/26 18:58:45.709 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:45.808 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:45 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 {"id":"5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93358e 5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 {"id":"5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93358e 5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 {"id":"5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93358e 5d27e0c8-4bb3-4899-b60a-ff72a1c2b1e9 /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:45.999 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:58:46.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:46.001 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:46.101 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:46.203 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later [2024/04/26 18:58:46.218 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:58:46.224 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/04/26 18:58:46.308 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/04/26 18:58:46.402 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:58:46.414 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:46.507 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:58:46.512 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:58:46.599 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:58:46.698 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:46.706 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:58:46.710 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:46.807 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/04/26 18:58:46.812 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/04/26 18:58:46.813 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/04/26 18:58:46.816 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/04/26 18:58:46.906 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000f19128 TaskPositions:map[cf819cb4-1e85-4662-8dd7-912389e7ff67:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf819cb4-1e85-4662-8dd7-912389e7ff67 {"id":"cf819cb4-1e85-4662-8dd7-912389e7ff67","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7beb cf819cb4-1e85-4662-8dd7-912389e7ff67 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/cf819cb4-1e85-4662-8dd7-912389e7ff67/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000f19128 TaskPositions:map[cf819cb4-1e85-4662-8dd7-912389e7ff67:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf819cb4-1e85-4662-8dd7-912389e7ff67 {"id":"cf819cb4-1e85-4662-8dd7-912389e7ff67","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7beb cf819cb4-1e85-4662-8dd7-912389e7ff67 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/cf819cb4-1e85-4662-8dd7-912389e7ff67/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000f19128 TaskPositions:map[cf819cb4-1e85-4662-8dd7-912389e7ff67:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf819cb4-1e85-4662-8dd7-912389e7ff67 {"id":"cf819cb4-1e85-4662-8dd7-912389e7ff67","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7beb cf819cb4-1e85-4662-8dd7-912389e7ff67 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERR+ grep -q 'etcd info' OR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/cf819cb4-1e85-4662-8dd7-912389e7ff67/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2024/04/26 18:58:47.104 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:47.298 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:58:47.319 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:58:47.324 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:47.400 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 9-th check, retry later check diff successfully [2024/04/26 18:58:47.501 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:58:47.600 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:47.601 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/04/26 18:58:47.615 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/04/26 18:58:47.722 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:47.796 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:47.814 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:47.896 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:47.896 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:47 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> [2024/04/26 18:58:48.006 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:48.008 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:58:48.009 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:48.121 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:48.303 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:48.415 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:48.512 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:48.512 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:48.514 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:48.595 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/04/26 18:58:48.599 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/04/26 18:58:48.699 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table changefeed_pause_resume.t3 exists [2024/04/26 18:58:48.795 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/04/26 18:58:48.796 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:48.805 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/04/26 18:58:48.913 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:48.915 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:48.997 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:49.008 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:58:49.015 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:49.299 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:49.300 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:49.317 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:49.505 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:49.603 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:49.706 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore table test.finish_mark not exists for 10-th check, retry later [2024/04/26 18:58:49.807 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:49.812 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:49.896 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:49.910 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:58:50.015 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:50.019 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:50.106 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/04/26 18:58:50.201 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:50.204 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:50.207 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:50.213 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:58:50.214 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:50.305 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/04/26 18:58:50.306 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/04/26 18:58:50.311 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/04/26 18:58:50.311 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/04/26 18:58:50.402 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:50.408 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:50.411 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:58:50.607 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:50.703 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:58:50.895 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:50.916 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3918000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23870, start at 2024-04-26 18:58:49.04150003 +0800 CST m=+4.349071965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log [2024/04/26 18:58:51.114 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:51.195 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:58:51.199 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check diff failed 1-th time, retry later [2024/04/26 18:58:51.318 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:51.396 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/04/26 18:58:51.421 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/04/26 18:58:51.515 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:51.602 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:58:51.604 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:51.606 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:58:51.610 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:58:51.698 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:51.716 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table test.finish_mark not exists for 11-th check, retry later [2024/04/26 18:58:51.241 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/04/26 18:58:51.523 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 18:58:51.669 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 18:58:51.735 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 18:58:51.738 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 18:58:51.940 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 18:58:51.941 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 18:58:51.819 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:52.004 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:52.011 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:52.023 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:52.013 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 18:58:52.139 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 18:58:52.098 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/04/26 18:58:52.123 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:58:52.202 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:58:52.209 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:52.302 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53bc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36318, start at 2024-04-26 18:58:50.77534237 +0800 CST m=+4.742686365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log [2024/04/26 18:58:52.208 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 18:58:52.211 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 18:58:52.505 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:52.514 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:52.518 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:58:52.519 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:52.523 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:52.457 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 18:58:52.458 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 18:58:52.571 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 18:58:52.677 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:52.621 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:52.710 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:52.744 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 18:58:52.746 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 18:58:52.863 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 18:58:52.863 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 18:58:52.906 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] Starting CDC state checker... [2024/04/26 18:58:52.905 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:52.920 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:53.013 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:58:53.038 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 18:58:53.082 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 18:58:53.084 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:58:53.106 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:53.305 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] check diff failed 2-th time, retry later [2024/04/26 18:58:53.222 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 18:58:53.223 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 18:58:53.269 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 18:58:53.400 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 18:58:53.448 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 18:58:53.450 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 18:58:53.517 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/04/26 18:58:53.571 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 18:58:53.571 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 18:58:53.610 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] table test.finish_mark not exists for 12-th check, retry later [2024/04/26 18:58:53.602 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/04/26 18:58:53.698 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 18:58:53.708 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:58:53.717 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:53.800 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 18:58:53.742 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 18:58:53.791 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 18:58:53.793 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 18:58:53.918 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 18:58:53.920 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 18:58:53.970 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 18:58:53.931 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 18:58:53.937 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:58:54.007 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 18:58:54.032 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/04/26 18:58:54.093 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 18:58:54.226 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 18:58:54.229 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 18:58:54.235 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/04/26 18:58:54.306 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 18:58:54.338 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:58:54.359 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 18:58:54.360 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 18:58:54.460 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd305acae_da1c_4861_89c3_f7caec5be867"] [2024/04/26 18:58:54.509 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/04/26 18:58:54.544 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:54.555 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 18:58:54.755 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 18:58:54.655 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/04/26 18:58:54.710 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb0634982_8117_4496_b9c2_97c994dbcb94"] [2024/04/26 18:58:54.795 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 18:58:54.796 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 18:58:55.002 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 18:58:55.003 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 18:58:54.902 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs92593238_e8da_4e73_8e24_dd97fbe53572"] [2024/04/26 18:58:55.100 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs08bf8193_2fb8_458e_8d4d_f4658806f7f3"] [2024/04/26 18:58:55.051 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 18:58:55.171 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 18:58:55.214 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 18:58:55.215 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 18:58:55.410 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 18:58:55.411 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 18:58:55.460 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] table test.finish_mark not exists for 13-th check, retry later [2024/04/26 18:58:55.400 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4bc90e18_8648_4d38_934d_115a61920c7b"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:58:55.586 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 18:58:55.635 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 18:58:55.638 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 18:58:55.755 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/04/26 18:58:55.756 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] check diff failed 3-th time, retry later [2024/04/26 18:58:55.805 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/04/26 18:58:55.812 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/04/26 18:58:55.897 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:55.898 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs62e9d01e_60ed_437a_a706_90bf0e0d74c9"] [2024/04/26 18:58:55.900 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:55.795 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 18:58:55.927 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/04/26 18:58:55.968 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 18:58:55.970 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/04/26 18:58:55.998 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8358e2da_a4b0_4e4c_9e1c_e82bff9cb4f4"] [2024/04/26 18:58:56.212 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:58:56.714 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:56.817 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:56.901 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:56.919 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24401) - No such process wait process 24401 exit for 1-th time... process 24401 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 18:58:55 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2456324565.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:58:57.096 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:57.206 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:57.429 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:57.505 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table test.finish_mark not exists for 14-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:58:57.708 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:57.709 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:57.710 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:57.721 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:57.728 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] pass test: get tso cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:56 CST 2024] <<<<<< run test case http_api success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= [2024/04/26 18:58:57.811 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:57.811 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0029c688_b3f0_4af2_bb06_dd8130c4092c"] [2024/04/26 18:58:57.814 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/04/26 18:58:58.003 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:58.021 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:58.110 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/04/26 18:58:58.198 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:58.202 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:58.210 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba8680001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26351, start at 2024-04-26 18:58:56.170098113 +0800 CST m=+5.992629333 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:56.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:56.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:56.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 18:58:58.311 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:58.419 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:58:58.604 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:58:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000c9f3b0 TaskPositions:map[d64a3e21-c359-4fe9-b5e2-2061f8d82be1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d64a3e21-c359-4fe9-b5e2-2061f8d82be1 {"id":"d64a3e21-c359-4fe9-b5e2-2061f8d82be1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c1e d64a3e21-c359-4fe9-b5e2-2061f8d82be1 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/d64a3e21-c359-4fe9-b5e2-2061f8d82be1/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000c9f3b0 TaskPositions:map[d64a3e21-c359-4fe9-b5e2-2061f8d82be1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d64a3e21-c359-4fe9-b5e2-2061f8d82be1 {"id":"d64a3e21-c359-4fe9-b5e2-2061f8d82be1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c1e d64a3e21-c359-4fe9-b5e2-2061f8d82be1 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/d64a3e21-c359-4fe9-b5e2-2061f8d82be1/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc000c9f3b0 TaskPositions:map[d64a3e21-c359-4fe9-b5e2-2061f8d82be1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d64a3e21-c359-4fe9-b5e2-2061f8d82be1 {"id":"d64a3e21-c359-4fe9-b5e2-2061f8d82be1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c1e d64a3e21-c359-4fe9-b5e2-2061f8d82be1 /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/d64a3e21-c359-4fe9-b5e2-2061f8d82be1/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:58:58.905 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:58.906 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:58.916 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:58:59.020 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:59.101 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:59.114 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/04/26 18:58:59.115 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:58:59.124 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:58:59.126 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:58:59.303 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:59.311 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:59.497 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:59.514 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:59.523 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:59.611 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:59.612 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:59.715 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:58:59.822 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:58:59.824 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table test.finish_mark not exists for 15-th check, retry later [2024/04/26 18:58:59.914 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:00.102 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff successfully [2024/04/26 18:59:00.203 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:00.305 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:00.314 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table big_txn.finish_mark_1 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:58:48 CST 2024] <<<<<< run test case big_txn success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:58:57 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> [2024/04/26 18:59:00.395 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:00.498 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:00.507 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:00.628 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:00.721 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:00.721 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:00.804 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:00.810 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:59:00 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:01.010 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:01.129 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] The 1 times to try to start tidb cluster... [2024/04/26 18:59:01.226 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:01.309 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:01.424 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting CDC state checker... [2024/04/26 18:59:01.524 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:01.605 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:01.629 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table test.finish_mark exists check diff successfully [2024/04/26 18:59:01.723 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:01.808 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:01.808 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:02.005 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:02.108 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:02.111 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:02.131 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:02.232 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:02.495 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:02.504 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:02.513 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:59:02 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/04/26 18:59:02.344 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/04/26 18:59:02.523 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 18:59:02.528 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 18:59:02.601 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:02.676 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 18:59:02.678 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 18:59:02.685 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 18:59:02.686 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 18:59:02.795 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:02.898 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:02.907 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:03.010 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:03.023 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:02.833 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 18:59:02.842 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 18:59:02.996 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 18:59:02.997 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 18:59:03.017 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 18:59:03.017 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 18:59:03.095 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.102 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.224 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.149 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 18:59:03.177 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 18:59:03.299 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 18:59:03.301 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 18:59:03.322 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 18:59:03.323 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:03.313 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 28.861676366s"] [2024/04/26 18:59:03.314 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.401 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:03.407 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:03.424 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:03.517 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.518 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:03.526 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:03.447 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 18:59:03.468 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff failed 1-th time, retry later [2024/04/26 18:59:03.601 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:03.621 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 18:59:03.623 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 18:59:03.648 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 18:59:03.649 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 18:59:03.771 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 18:59:03.795 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 449348659195150347 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Fri Apr 26 18:58:53 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Fri Apr 26 18:58:53 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml check diff failed 1-th time, retry later check diff failed 2-th time, retry later [2024/04/26 18:59:03.913 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 18:59:03.915 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 18:59:03.933 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 18:59:03.934 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 18:59:04.059 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 18:59:04.085 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 18:59:04.203 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 18:59:04.204 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 18:59:04.230 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 18:59:04.230 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 18:59:04.096 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:04.234 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:04.297 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:04.318 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:59:04 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> [2024/04/26 18:59:04.351 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 18:59:04.386 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 18:59:04.500 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 18:59:04.502 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 18:59:04.535 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 18:59:04.536 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 18:59:04.408 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:04.498 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:04.500 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:04.650 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 18:59:04.703 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 18:59:04.817 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 18:59:04.818 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 18:59:04.614 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:04.619 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:04.703 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:04.709 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:04.797 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:04.800 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:04.801 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:04.806 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:04.819 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c13700016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22294, start at 2024-04-26 18:59:03.040474773 +0800 CST m=+4.431771599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:03.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:03.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:03.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log [2024/04/26 18:59:04.861 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 18:59:04.861 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 18:59:04.977 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 18:59:05.023 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 18:59:05.101 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:05.160 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 18:59:05.162 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 18:59:05.210 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 18:59:05.210 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:05.308 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:05.313 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff failed 2-th time, retry later [2024/04/26 18:59:05.441 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 18:59:05.500 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 18:59:05.598 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 18:59:05.600 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/04/26 18:59:05.497 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:05.603 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:05.610 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:05.616 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff successfully [Fri Apr 26 18:59:05 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> [2024/04/26 18:59:05.656 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/04/26 18:59:05.656 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 18:59:05.745 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/04/26 18:59:05.805 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/04/26 18:59:05.801 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:05.806 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) [2024/04/26 18:59:05.893 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/04/26 18:59:05.894 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/04/26 18:59:05.957 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/04/26 18:59:05.958 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/04/26 18:59:06.048 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/04/26 18:59:05.903 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:05.909 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:05.917 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:05.921 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:05.926 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:05.926 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:05.927 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:06.103 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) [2024/04/26 18:59:06.235 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/04/26 18:59:06.342 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/04/26 18:59:06.344 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/04/26 18:59:06.210 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:06.215 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... [2024/04/26 18:59:06.392 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/04/26 18:59:06.393 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/04/26 18:59:06.501 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/04/26 18:59:06.581 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/04/26 18:59:06.400 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:06.606 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:06.668 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/04/26 18:59:06.669 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/04/26 18:59:06.756 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/04/26 18:59:06.757 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/04/26 18:59:06.698 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:06.712 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:07.012 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:07.020 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:07.104 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.116 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:07.120 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.121 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c3b98000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19865, start at 2024-04-26 18:59:05.594788283 +0800 CST m=+4.653874995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log [2024/04/26 18:59:07.205 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:07.206 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.213 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.308 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:07.323 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Starting CDC state checker... [2024/04/26 18:59:07.414 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.505 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:07.601 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:07.605 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 18:59:07 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:07.695 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 3-th time, retry later [2024/04/26 18:59:08.016 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:08.102 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:08.211 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:08.300 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:08.312 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:08.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:08.402 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Verifying downstream PD is started... [2024/04/26 18:59:08.434 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:08.502 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:08.522 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:08.598 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24568) - No such process wait process 24568 exit for 1-th time... process 24568 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 18:59:07 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2472424726.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:59:08.718 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:08.719 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:08.805 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:08.807 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:08.830 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:09.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:09.009 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:09.405 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:09.497 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:09.504 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:09.523 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting CDC state checker... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later [2024/04/26 18:59:09.723 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:09.804 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:09.813 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:09.820 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:09.914 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:09.921 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:10.012 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:10.026 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:10.198 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:10.199 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:10.211 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:10.220 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:10.506 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:10.720 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:59:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0006bb2a8 TaskPositions:map[20286225-0aa2-465d-9226-c4930872bf0b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20286225-0aa2-465d-9226-c4930872bf0b {"id":"20286225-0aa2-465d-9226-c4930872bf0b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c5e 20286225-0aa2-465d-9226-c4930872bf0b /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/20286225-0aa2-465d-9226-c4930872bf0b/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0006bb2a8 TaskPositions:map[20286225-0aa2-465d-9226-c4930872bf0b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20286225-0aa2-465d-9226-c4930872bf0b {"id":"20286225-0aa2-465d-9226-c4930872bf0b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c5e 20286225-0aa2-465d-9226-c4930872bf0b /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/20286225-0aa2-465d-9226-c4930872bf0b/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0006bb2a8 TaskPositions:map[20286225-0aa2-465d-9226-c4930872bf0b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/20286225-0aa2-465d-9226-c4930872bf0b {"id":"20286225-0aa2-465d-9226-c4930872bf0b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c5e 20286225-0aa2-465d-9226-c4930872bf0b /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663282761731,"admin-job-type":0} /tidb/cdc/default/default/task/position/20286225-0aa2-465d-9226-c4930872bf0b/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 + grep -q 'etcd info' {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:10.795 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:10.927 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:11.006 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:11.008 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:11.019 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:11.107 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:11.195 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:11.203 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:11.301 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:11.304 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:11.306 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:11.318 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:11.415 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:11.599 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:11.611 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff failed 5-th time, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:12.102 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:12.108 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:12.122 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:12.221 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:12.299 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:59:12.307 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:12.316 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:12.318 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:12.332 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:12.416 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:12.705 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:12.808 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:13.103 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:13.298 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:13.314 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:13.314 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:13.354 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:13.517 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsba080c2f_83fc_4ba3_b365_33ee95108877"] [2024/04/26 18:59:13.550 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:13.745 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:13.802 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse7a379bd_43d4_4087_a119_b5f0ad9c0464"] [2024/04/26 18:59:13.843 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:13.899 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3918000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23870, start at 2024-04-26 18:58:49.04150003 +0800 CST m=+4.349071965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:14.009 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:14.015 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse849fce7_1002_4d8c_95bc_f42e6ab29ef2"] check diff successfully [2024/04/26 18:59:14.209 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs46719af6_a5a1_4275_9bcc_7181b20ee443"] [2024/04/26 18:59:14.413 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbce3d0c7_2d9d_4b5f_96f3_8c5c91db3242"] [2024/04/26 18:59:14.515 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1ecf133e_7624_4d8d_a436_89b0ac4a3435"] [2024/04/26 18:59:14.617 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:14.702 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:14.705 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3231726d_173e_4a45_85d9_21d60d01aef1"] [2024/04/26 18:59:14.715 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:14.718 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:14.895 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:14.038 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/04/26 18:59:14.633 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 18:59:14.638 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 18:59:14.916 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 18:59:14.917 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 18:59:14.918 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 18:59:14.919 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... [2024/04/26 18:59:15.000 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Verifying downstream PD is started... [2024/04/26 18:59:15.304 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:15.422 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:15.488 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 18:59:15.491 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:15.615 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:15.627 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check diff failed 1-th time, retry later [2024/04/26 18:59:15.708 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:15.709 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:15.800 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:15.829 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:15.904 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:15.904 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:15.908 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:15.920 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:15.775 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 18:59:15.776 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 18:59:15.776 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 18:59:15.778 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 18:59:16.002 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:16.107 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:16.331 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 18:59:16.342 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 18:59:16.500 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 18:59:16.502 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 18:59:16.507 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 18:59:16.508 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:16.515 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:16.606 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:16.778 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 18:59:16.806 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 18:59:16.806 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:16.900 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:16.903 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:16.915 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:16.938 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:17.003 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:17.007 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:17.022 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:17.068 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 18:59:17.070 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 18:59:17.024 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:17.097 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:17.102 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:17.118 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:17.096 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 18:59:17.097 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 18:59:17.250 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 18:59:17.280 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:17.408 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:17.409 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:17.588 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 18:59:17.590 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:17.699 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf7452742_f515_464f_9e42_7dd3577d8903"] [2024/04/26 18:59:17.631 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 18:59:17.633 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 18:59:17.849 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 18:59:17.898 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:18.017 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:17.901 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 18:59:18.057 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 18:59:18.058 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:18.104 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:18.203 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:18.205 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:18.213 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:18.223 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:18.173 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 18:59:18.174 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 18:59:18.225 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:18.301 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:18.323 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:18.326 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:18.326 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:18.416 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:18.441 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 18:59:18.497 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 18:59:18.498 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 18:59:18.607 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 18:59:18.609 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 18:59:18.597 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:18.618 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:18.718 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:18.723 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:18.919 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:18.962 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:19.210 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:19.320 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:19.398 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:19.406 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:19.423 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:19.423 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:19.509 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:19.519 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:19.620 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:19.621 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:19.722 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:19.816 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:19.915 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:19.922 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:20.013 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:20.014 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53bc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36318, start at 2024-04-26 18:58:50.77534237 +0800 CST m=+4.742686365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:20.221 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] skip some SQLs in tidb v4.0.x table common_1.v1 not exists for 1-th check, retry later table common_1.v1 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (24729) - No such process wait process 24729 exit for 1-th time... process 24729 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 18:59:19 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.2488624888.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:59:20.595 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:20.603 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:20.609 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:20.617 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:20.622 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:20.704 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:20.707 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:20.816 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:20.817 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:20.922 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:20.923 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:21.025 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:21.126 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:21.131 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:21.208 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:21.313 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d19180019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32978, start at 2024-04-26 18:59:19.799342253 +0800 CST m=+4.569540377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:21.703 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:21.714 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:21.797 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:21.816 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:21.901 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:21.913 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff failed 4-th time, retry later [2024/04/26 18:59:22.007 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:22.109 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:22.111 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:22.115 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:22.209 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:22.220 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:22.326 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:22.418 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:22.422 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:59:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0012c17b8 TaskPositions:map[11a97059-3906-4f74-9c89-e6b06bccb6bf:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing tableID: 84, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11a97059-3906-4f74-9c89-e6b06bccb6bf {"id":"11a97059-3906-4f74-9c89-e6b06bccb6bf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c95 11a97059-3906-4f74-9c89-e6b06bccb6bf /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663413833761,"admin-job-type":0} /tidb/cdc/default/default/task/position/11a97059-3906-4f74-9c89-e6b06bccb6bf/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0012c17b8 TaskPositions:map[11a97059-3906-4f74-9c89-e6b06bccb6bf:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing tableID: 84, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11a97059-3906-4f74-9c89-e6b06bccb6bf {"id":"11a97059-3906-4f74-9c89-e6b06bccb6bf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c95 11a97059-3906-4f74-9c89-e6b06bccb6bf /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1+ grep -q 'failed to get info:' 800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663413833761,"admin-job-type":0} /tidb/cdc/default/default/task/position/11a97059-3906-4f74-9c89-e6b06bccb6bf/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {ClusterID:default ID:default/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 Info:{"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} Status:0xc0012c17b8 TaskPositions:map[11a97059-3906-4f74-9c89-e6b06bccb6bf:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing tableID: 84, resolvedTs: 449348663282761731, checkpointTs: 449348663282761731, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11a97059-3906-4f74-9c89-e6b06bccb6bf {"id":"11a97059-3906-4f74-9c89-e6b06bccb6bf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0d7d7c95 11a97059-3906-4f74-9c89-e6b06bccb6bf /tidb/cdc/default/default/changefeed/info/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"upstream-id":7362127572630951477,"namespace":"default","changefeed-id":"42a163e6-e86a-4a1e-9e9e-2874d69bf6d9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T18:57:35.498901544+08:00","start-ts":449348647108739073,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348647121584131} /tidb/cdc/default/default/changefeed/status/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":449348663282761731,"min-table-barrier-ts":449348663413833761,"admin-job-type":0} /tidb/cdc/default/default/task/position/11a97059-3906-4f74-9c89-e6b06bccb6bf/42a163e6-e86a-4a1e-9e9e-2874d69bf6d9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362127572630951477 {"id":7362127572630951477,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later [2024/04/26 18:59:22.503 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:22.695 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:22.701 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Verifying downstream PD is started... [2024/04/26 18:59:22.809 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:22.901 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:23.097 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:23.105 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:23.211 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:23.297 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:23.303 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:23.311 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:23.318 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:23.326 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:23.427 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba8680001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26351, start at 2024-04-26 18:58:56.170098113 +0800 CST m=+5.992629333 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:56.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:56.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:56.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:23.521 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:23.607 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:23.617 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:23.712 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:23.713 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d3f3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31189, start at 2024-04-26 18:59:22.196607115 +0800 CST m=+6.733574751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:23.803 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:23.811 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:23.917 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 5-th time, retry later Starting CDC state checker... table common_1.recover_and_insert not exists for 2-th check, retry later [2024/04/26 18:59:24.007 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:24.200 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:24.204 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:24.316 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:24.395 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:24.408 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:24.503 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:24.517 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:24.616 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ed00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:26023, start at 2024-04-26 18:59:23.236260197 +0800 CST m=+4.421423053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 18:59:24.798 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:24.812 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:24.816 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:24.824 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:24.919 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:25.004 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table kill_owner_with_ddl.t3 exists check diff failed 1-th time, retry later Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:25.122 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:25.210 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:25.299 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:25.318 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:25.320 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:25.511 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:25.038 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/04/26 18:59:25.348 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 18:59:25.352 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 18:59:25.510 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 18:59:25.511 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 18:59:25.523 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 18:59:25.526 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 18:59:25.693 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 18:59:25.595 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:25.699 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:25.703 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 6-th time, retry later [2024/04/26 18:59:25.710 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 18:59:25.955 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 18:59:25.808 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:25.918 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:25.922 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:26.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:26.024 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:26.019 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 18:59:26.054 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 18:59:26.056 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] Starting CDC state checker... [2024/04/26 18:59:26.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:26.121 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:26.286 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 18:59:26.314 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 18:59:26.309 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:26.417 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:26.418 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:26.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table common_1.recover_and_insert not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20222, start at 2024-04-26 18:59:25.812591896 +0800 CST m=+5.583901411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log [2024/04/26 18:59:26.578 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 18:59:26.581 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 18:59:26.598 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 18:59:26.600 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:26.597 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:26.717 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:26.719 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:26.829 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff successfully [2024/04/26 18:59:26.903 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:26.912 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:27.004 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Starting CDC state checker... [2024/04/26 18:59:26.991 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 18:59:27.096 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 18:59:27.099 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:59:27 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> [2024/04/26 18:59:27.116 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:27.117 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:27.122 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:27.207 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:27.375 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 18:59:27.382 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 18:59:27.387 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 18:59:27.312 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:27.317 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:27.414 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:27.417 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:27.500 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:27.509 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:27.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:27.813 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:27.761 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 18:59:27.763 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 18:59:27.797 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 18:59:27.956 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 18:59:28.040 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 18:59:27.814 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:28.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:28.024 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:28.043 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 18:59:28.150 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 18:59:28.151 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 18:59:28.285 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 18:59:28.108 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:28.217 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:28.301 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:28.319 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c13700016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22294, start at 2024-04-26 18:59:03.040474773 +0800 CST m=+4.431771599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:03.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:03.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:03.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully [2024/04/26 18:59:28.509 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later Starting CDC state checker... [2024/04/26 18:59:28.411 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:28.504 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:28.688 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 18:59:28.695 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 18:59:28.613 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:28.615 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:28.615 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:28.624 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:28.723 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:28.808 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:28.850 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 18:59:28.853 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 18:59:29.009 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 18:59:28.900 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:28.902 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:29.010 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:29.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:29.215 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 18:59:29.313 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:29.443 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 18:59:29.447 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:29.415 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:29.522 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:29.522 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:29.596 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check diff failed 1-th time, retry later [2024/04/26 18:59:29.659 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 18:59:29.663 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 18:59:29.699 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:29.895 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c3b98000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19865, start at 2024-04-26 18:59:05.594788283 +0800 CST m=+4.653874995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:29.911 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:29.912 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:30.013 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:30.229 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:30.404 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:30.407 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table common_1.finish_mark not exists for 2-th check, retry later [2024/04/26 18:59:30.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:30.507 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:30.534 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:31.007 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:31.097 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:31.137 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:59:31.154 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:59:31.210 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa14380b2_88e9_43ea_9eae_19d7bfb979c8"] [2024/04/26 18:59:31.309 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5835ce20_3b35_4981_bc2f_c89a432f7d78"] [2024/04/26 18:59:31.316 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:31.405 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2dacdb09_a5bf_4528_9b7b_feb88fffbbab"] [2024/04/26 18:59:31.434 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:31.599 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb5129467_3a1e_41a5_98a6_9438f49e26e0"] [2024/04/26 18:59:31.610 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1ff0e5b9_6bdc_4c4b_8804_63ec0c7f19b4"] [2024/04/26 18:59:31.716 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs992b7c5e_837b_4e8e_a0c9_e875130fa91f"] [2024/04/26 18:59:32.005 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:32.006 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.218 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.218 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] check diff failed 2-th time, retry later table common_1.finish_mark not exists for 3-th check, retry later [2024/04/26 18:59:32.226 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:32.397 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:32.403 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:32.417 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.502 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:32.507 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:32.511 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.603 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:32.607 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.612 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:32.898 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs568722ff_029d_466c_92bd_5d89f180c640"] [2024/04/26 18:59:32.906 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:33.007 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd2500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22262, start at 2024-04-26 18:59:31.649243706 +0800 CST m=+4.283901169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3904000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23776, start at 2024-04-26 18:58:49.037929969 +0800 CST m=+4.426558104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.045 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b3918000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:23870, start at 2024-04-26 18:58:49.04150003 +0800 CST m=+4.349071965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:49.048 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:49.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:49.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:59:33 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.2802428026.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:59:33.397 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:33.398 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:33.533 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log [2024/04/26 18:59:33.642 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:33.645 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:33.705 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:33.717 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:33.796 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:33.811 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:33.814 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:33.910 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:33.914 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:33.918 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:34.020 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table common_1.finish_mark exists check diff failed 3-th time, retry later Verifying downstream PD is started... check diff successfully start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:34.595 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:34.616 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:34.800 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:34.901 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:34.910 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:34.911 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:34.929 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:34.931 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:59:34 CST 2024] <<<<<< run test case common_1 success! >>>>>> [2024/04/26 18:59:35.014 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:35.018 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:35.099 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:35.105 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:35.110 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:35.126 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:35.333 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:35.600 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:35.732 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:35.917 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:35.935 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:35.940 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff failed 4-th time, retry later Starting CDC state checker... [2024/04/26 18:59:36.002 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:36.021 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:36.105 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:36.118 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:36.123 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:36.128 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:36.216 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:36.306 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs97efd6b3_674e_40d8_8a61_168dd91c2aa8"] [2024/04/26 18:59:36.321 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:36.412 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:36.557 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m7.697999097s"] [2024/04/26 18:59:36.629 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:59:36 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.28137.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 [2024/04/26 18:59:36.731 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Create changefeed successfully! ID: 40969f45-4e63-4efb-a62c-2db38c1cf92a Info: {"upstream_id":7362128517417682748,"namespace":"default","id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T18:59:36.884149228+08:00","start_ts":449348678912311302,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348678912311302,"checkpoint_ts":449348678912311302,"checkpoint_time":"2024-04-26 18:59:36.759"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:59:37.026 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:37.118 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:37.201 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:37.208 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x table mark.finish_mark_4 not exists for 1-th check, retry later [2024/04/26 18:59:37.313 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:37.404 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:37.408 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:37.419 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:37.424 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:37.500 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:37.512 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:37.528 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:37.597 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:37.606 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Verifying downstream PD is started... [2024/04/26 18:59:37.807 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:37.824 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:38.096 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:38.131 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:38.140 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:38.197 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:38.228 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:38.416 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:38.418 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:38.498 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:38.504 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:38.511 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:38.520 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] check diff failed 5-th time, retry later [2024/04/26 18:59:38.603 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:38.629 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:38.710 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... [2024/04/26 18:59:38.911 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:38.999 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later [2024/04/26 18:59:39.226 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:39.314 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:39.400 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:39.401 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:39.415 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:39.536 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:39.601 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:39.601 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:39.610 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:39.617 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:39.617 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:39.708 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:39.727 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:39.803 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53b40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36259, start at 2024-04-26 18:58:50.734198294 +0800 CST m=+4.787295858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683b53bc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:36318, start at 2024-04-26 18:58:50.77534237 +0800 CST m=+4.742686365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:50.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:50.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:50.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.40248.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 18:59:40.008 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:40.118 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:40.323 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff successfully + set +x + tso='449348679791804417 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348679791804417 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/04/26 18:59:40.418 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:40.423 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:40.506 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:40.512 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:40.696 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:40.700 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:40.701 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:40.708 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:40.716 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:40.725 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:40.803 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:40.823 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:40.833 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:40.995 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:41.110 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:41.415 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:41.498 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:41.505 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:41.520 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:41.530 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table mark.finish_mark not exists for 2-th check, retry later check diff failed 1-th time, retry later [2024/04/26 18:59:41.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:41.728 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:41.807 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:41.818 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:41.820 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:41.827 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:41.915 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:41.921 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:41.922 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:42.102 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:42.112 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ***************** properties ***************** "recordcount"="100" "scanproportion"="0" "workload"="core" "fieldlengthdistribution"="constant" "readproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "fieldlength"="15000" "requestdistribution"="uniform" "operationcount"="0" "dotransactions"="false" "threadcount"="10" "mysql.port"="4000" "updateproportion"="0" "insertproportion"="0" "mysql.user"="root" "fieldcount"="1000" "mysql.db"="consistent_replicate_storage_file_large_value" ********************************************** Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:43.107 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:43.110 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:43.122 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:43.208 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:43.213 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:43.313 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:43.316 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:43.397 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:43.423 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:43.425 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:43.503 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:43.512 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:43.526 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:43.602 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:43.612 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:43.615 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table mark.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba5440018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26256, start at 2024-04-26 18:58:55.985545433 +0800 CST m=+6.107822657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:55.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:55.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:55.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ba8680001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:26351, start at 2024-04-26 18:58:56.170098113 +0800 CST m=+5.992629333 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:00:56.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:58:56.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:48:56.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30483.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 18:59:44.003 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:44.006 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:44.016 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:44.107 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:44.112 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + set +x + tso='449348680764882945 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348680764882945 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 2-th time, retry later [2024/04/26 18:59:44.201 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:44.212 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:44.314 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:44.316 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:44.324 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:44.397 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:44.404 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d19180019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32978, start at 2024-04-26 18:59:19.799342253 +0800 CST m=+4.569540377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:44.501 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:44.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:44.919 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:44.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:45.000 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:45.126 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:45.199 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 18:59:45.201 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:45.205 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:45.308 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:45.327 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:45.401 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:45.403 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:45.408 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:45.414 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table mark.finish_mark not exists for 4-th check, retry later [2024/04/26 18:59:45.711 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:45.721 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:45.799 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:46.007 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:46.101 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:46.104 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:46.108 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:46.298 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:46.302 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:46.308 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:46.309 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:46.495 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:46.512 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:46.522 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ed00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:26023, start at 2024-04-26 18:59:23.236260197 +0800 CST m=+4.421423053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:46.615 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/04/26 18:59:46.639 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:46.827 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:46.828 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:46.831 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:46.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:47.000 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:47.008 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 18:59:47.043 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/04/26 18:59:47.102 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:47.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/04/26 18:59:47.232 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/04/26 18:59:47.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20222, start at 2024-04-26 18:59:25.812591896 +0800 CST m=+5.583901411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:47.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/04/26 18:59:47.516 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/04/26 18:59:47.550 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:47.719 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:47.719 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:47.930 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4ed0b80a_2684_4aa5_9502_14737eddcb70"] [2024/04/26 18:59:47.950 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3594f305_8bd3_4792_8f55_4088e7bbd6ba"] [2024/04/26 18:59:47.953 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] check diff failed 4-th time, retry later [2024/04/26 18:59:48.116 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/04/26 18:59:48.137 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs98314588_03bc_4af0_8a82_b695cfbf2517"] [2024/04/26 18:59:48.224 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6d0d2828_a533_4274_8468_2db01d1e2ed6"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d3f3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31189, start at 2024-04-26 18:59:22.196607115 +0800 CST m=+6.733574751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:48.439 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:48.505 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:48.509 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:48.541 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c12c0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22214, start at 2024-04-26 18:59:02.973322955 +0800 CST m=+4.416542939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:02.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:02.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:02.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c13700016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:22294, start at 2024-04-26 18:59:03.040474773 +0800 CST m=+4.431771599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:03.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:03.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:03.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:59:48 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.2625426256.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:59:48.625 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:48.630 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:48.653 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:48.657 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:48.805 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7021ba45_786d_440b_98b2_69f3938dd915"] [2024/04/26 18:59:48.815 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:48.910 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:48.913 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:49.000 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table mark.finish_mark not exists for 6-th check, retry later [2024/04/26 18:59:49.113 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:49.115 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:49.198 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:49.204 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 18:59:49.311 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 18:59:49.316 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:49.321 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 18:59:49.423 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:49.428 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:49.516 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c39280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19795, start at 2024-04-26 18:59:05.439932104 +0800 CST m=+4.611064623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683c3b98000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:19865, start at 2024-04-26 18:59:05.594788283 +0800 CST m=+4.653874995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:05.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:05.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:05.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 18:59:49 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2373623738.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 18:59:49.623 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:49.631 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:49.715 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:49.725 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:49.827 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 18:59:49.835 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 5-th time, retry later [2024/04/26 18:59:49.896 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:50.010 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:50.012 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:50.198 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:50.316 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:50.321 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 18:59:50.413 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:50.426 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:50.598 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 18:59:50.603 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 18:59:50.608 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:50.630 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:50.698 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:50.810 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:50.920 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:50.929 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:51.016 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:51.106 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/04/26 18:59:51.210 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 18:59:51.219 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 18:59:51.221 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:51.311 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:51.319 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:59:51 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67c6285-dfba-4938-8de0-ccad3e77b92d {"id":"c67c6285-dfba-4938-8de0-ccad3e77b92d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f8b748e c67c6285-dfba-4938-8de0-ccad3e77b92d /tidb/cdc/default/default/upstream/7362128578355833676 {"id":7362128578355833676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67c6285-dfba-4938-8de0-ccad3e77b92d {"id":"c67c6285-dfba-4938-8de0-ccad3e77b92d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f8b748e c67c6285-dfba-4938-8de0-ccad3e77b92d /tidb/cdc/default/default/upstream/7362128578355833676 {"id":7362128578355833676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67c6285-dfba-4938-8de0-ccad3e77b92d {"id":"c67c6285-dfba-4938-8de0-ccad3e77b92d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f8b748e c67c6285-dfba-4938-8de0-ccad3e77b92d /tidb/cdc/default/default/upstream/7362128578355833676 {"id":7362128578355833676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683eeb280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25988, start at 2024-04-26 18:59:49.614503933 +0800 CST m=+4.357896437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.578 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.578 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log [2024/04/26 18:59:51.507 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:51.613 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table mark.finish_mark exists check diff successfully [2024/04/26 18:59:51.707 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:51.720 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:51.826 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 18:59:51 CST 2024] <<<<<< run test case multi_source success! >>>>>> [2024/04/26 18:59:51.941 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 18:59:51.999 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:52.007 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 18:59:52.034 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:52.096 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] INSERT_ERROR - Takes(s): 8.4, Count: 67, OPS: 7.9, Avg(us): 852414, Min(us): 439776, Max(us): 1837947, 95th(us): 0, 99th(us): 0 [2024/04/26 18:59:52.209 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:52.329 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff successfully [2024/04/26 18:59:52.417 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:52.423 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:52.532 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:52.628 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 18:59:52.631 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:52.634 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 18:59:52.704 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:52.712 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:52.744 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 10:59:52 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1393a896-7a98-4cac-a606-6795e0ebcef8 {"id":"1393a896-7a98-4cac-a606-6795e0ebcef8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f918790 1393a896-7a98-4cac-a606-6795e0ebcef8 /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1393a896-7a98-4cac-a606-6795e0ebcef8 {"id":"1393a896-7a98-4cac-a606-6795e0ebcef8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f918790 1393a896-7a98-4cac-a606-6795e0ebcef8 /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1393a896-7a98-4cac-a606-6795e0ebcef8 {"id":"1393a896-7a98-4cac-a606-6795e0ebcef8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f918790 1393a896-7a98-4cac-a606-6795e0ebcef8 /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.23809.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7362128587817658600,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-04-26T18:59:52.75649859+08:00","start_ts":449348683071750149,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348683071750149,"checkpoint_ts":449348683071750149,"checkpoint_time":"2024-04-26 18:59:52.626"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [2024/04/26 18:59:52.906 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:52.913 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:52.932 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:53.028 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:53.129 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 18:59:53.133 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 18:59:53.138 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:53.206 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:53.220 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:53.246 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:53.408 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:53.409 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:53.431 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:53.597 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 18:59:53.721 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 18:59:53.800 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 18:59:53.823 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:53.909 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:53.910 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] check diff failed 1-th time, retry later Starting CDC state checker... [2024/04/26 18:59:54.003 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:54.211 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:54.215 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 18:59:54.299 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:54.327 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 18:59:54.497 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 18:59:54.513 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 18:59:54.603 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Run finished, takes 13.442440108s INSERT_ERROR - Takes(s): 11.9, Count: 100, OPS: 8.4, Avg(us): 773575, Min(us): 367219, Max(us): 1837947, 95th(us): 0, 99th(us): 0 [2024/04/26 18:59:54.704 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:54.707 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:54.799 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:54.916 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:54.924 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:54.944 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:54.962 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 18:59:55.027 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 18:59:55.037 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later [2024/04/26 18:59:55.235 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 18:59:55.245 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:55.316 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 18:59:55.317 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd2500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22262, start at 2024-04-26 18:59:31.649243706 +0800 CST m=+4.283901169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 18:59:55.610 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/04/26 18:59:55.616 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff failed 2-th time, retry later [2024/04/26 18:59:55.803 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m21.352304016s"] [2024/04/26 18:59:55.884 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m21.432647089s"] table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table mark.finish_mark_2 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table mark.finish_mark_2 not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f97e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30853, start at 2024-04-26 19:00:00.668017981 +0800 CST m=+4.451515180 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later table mark.finish_mark_2 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20161, start at 2024-04-26 18:59:25.837034913 +0800 CST m=+5.689649620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d77c40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:20222, start at 2024-04-26 18:59:25.812591896 +0800 CST m=+5.583901411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:25.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:25.809 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:25.809 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24078.out cli tso query --pd=http://127.0.0.1:2379 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d17880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32874, start at 2024-04-26 18:59:19.68433351 +0800 CST m=+4.544026359 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d19180019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:32978, start at 2024-04-26 18:59:19.799342253 +0800 CST m=+4.569540377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:19.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:19.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:19.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting CDC state checker... check diff successfully Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x + tso='449348686198079489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348686198079489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9CF6E407537C1 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:00:05 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ec00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:25927, start at 2024-04-26 18:59:23.19086387 +0800 CST m=+4.427051177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d4ed00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:26023, start at 2024-04-26 18:59:23.236260197 +0800 CST m=+4.421423053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:23.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:23.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:23.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:00:06 CST 2024] <<<<<< START cdc server in canal_json_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.3008230084.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_storage_file_large_value.t2 exists check diff successfully check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:00:08 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.3170531707.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d400c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31104, start at 2024-04-26 18:59:22.267643122 +0800 CST m=+6.873789170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683d3f3c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:31189, start at 2024-04-26 18:59:22.196607115 +0800 CST m=+6.733574751 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:22.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:22.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:22.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.35220.out cli tso query --pd=http://127.0.0.1:2379 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:09 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80909c5e-3d29-42a5-859b-77dd4938a8e9 {"id":"80909c5e-3d29-42a5-859b-77dd4938a8e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd10991 80909c5e-3d29-42a5-859b-77dd4938a8e9 /tidb/cdc/default/default/upstream/7362128660853684201 {"id":7362128660853684201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80909c5e-3d29-42a5-859b-77dd4938a8e9 {"id":"80909c5e-3d29-42a5-859b-77dd4938a8e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd10991 80909c5e-3d29-42a5-859b-77dd4938a8e9 /tidb/cdc/default/default/upstream/7362128660853684201 {"id":7362128660853684201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80909c5e-3d29-42a5-859b-77dd4938a8e9 {"id":"80909c5e-3d29-42a5-859b-77dd4938a8e9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd10991 80909c5e-3d29-42a5-859b-77dd4938a8e9 /tidb/cdc/default/default/upstream/7362128660853684201 {"id":7362128660853684201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.cli.30147.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: afd38b01-7f38-421a-b2aa-e6960aeb5c8f Info: {"upstream_id":7362128660853684201,"namespace":"default","id":"afd38b01-7f38-421a-b2aa-e6960aeb5c8f","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:00:09.481259218+08:00","start_ts":449348687465545730,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348687465545730,"checkpoint_ts":449348687465545730,"checkpoint_time":"2024-04-26 19:00:09.387"} PASS + set +x + tso='449348687450341380 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348687450341380 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:00:09 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.3525935261.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' Starting TLS TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 not exists for 8-th check, retry later check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/71c0cbf6-3ab7-412d-b285-28870fb34716 {ClusterID:default ID:default/71c0cbf6-3ab7-412d-b285-28870fb34716 Info:{"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} Status:0xc00184acf0 TaskPositions:map[17b7782e-e2f4-43aa-a406-064474f330e5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 88, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 94, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 82, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 91, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 84, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 89, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 87, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17b7782e-e2f4-43aa-a406-064474f330e5 {"id":"17b7782e-e2f4-43aa-a406-064474f330e5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93374c 17b7782e-e2f4-43aa-a406-064474f330e5 /tidb/cdc/default/default/changefeed/info/71c0cbf6-3ab7-412d-b285-28870fb34716 {"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} /tidb/cdc/default/default/changefeed/status/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":449348687564374021,"min-table-barrier-ts":449348687826518021,"admin-job-type":0} /tidb/cdc/default/default/task/position/17b7782e-e2f4-43aa-a406-064474f330e5/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/71c0cbf6-3ab7-412d-b285-28870fb34716 {ClusterID:default ID:default/71c0cbf6-3ab7-412d-b285-28870fb34716 Info:{"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} Status:0xc00184acf0 TaskPositions:map[17b7782e-e2f4-43aa-a406-064474f330e5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 88, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 94, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 82, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 91, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 84, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 89, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 87, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17b7782e-e2f4-43aa-a406-064474f330e5 {"id":"17b7782e-e2f4-43aa-a406-064474f330e5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93374c 17b7782e-e2f4-43aa-a406-064474f330e5 /tidb/cdc/default/default/changefeed/info/71c0cbf6-3ab7-412d-b285-28870fb34716 {"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} /tidb/cdc/default/default/changefeed/status/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":449348687564374021,"min-table-barrier-ts":449348687826518021,"admin-job-type":0} /tidb/cdc/default/default/task/position/17b7782e-e2f4-43aa-a406-064474f330e5/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/71c0cbf6-3ab7-412d-b285-28870fb34716 {ClusterID:default ID:default/71c0cbf6-3ab7-412d-b285-28870fb34716 Info:{"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} Status:0xc00184acf0 TaskPositions:map[17b7782e-e2f4-43aa-a406-064474f330e5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 92, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 88, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 94, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 82, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 91, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 84, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 89, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating tableID: 87, resolvedTs: 449348687564374022, checkpointTs: 449348687564374021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17b7782e-e2f4-43aa-a406-064474f330e5 {"id":"17b7782e-e2f4-43aa-a406-064474f330e5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0e93374c 17b7782e-e2f4-43aa-a406-064474f330e5 /tidb/cdc/default/default/changefeed/info/71c0cbf6-3ab7-412d-b285-28870fb34716 {"upstream-id":7362128284378597230,"namespace":"default","changefeed-id":"71c0cbf6-3ab7-412d-b285-28870fb34716","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:58:45.949193334+08:00","start-ts":449348665557385222,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348665583599622} /tidb/cdc/default/default/changefeed/status/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":449348687564374021,"min-table-barrier-ts":449348687826518021,"admin-job-type":0} /tidb/cdc/default/default/task/position/17b7782e-e2f4-43aa-a406-064474f330e5/71c0cbf6-3ab7-412d-b285-28870fb34716 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128284378597230 {"id":7362128284378597230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683eeb280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25988, start at 2024-04-26 18:59:49.614503933 +0800 CST m=+4.357896437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.578 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.578 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:12 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/313e378f-dddd-4cf0-99b3-6da63e659233 {"id":"313e378f-dddd-4cf0-99b3-6da63e659233","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fdd158e 313e378f-dddd-4cf0-99b3-6da63e659233 /tidb/cdc/default/default/upstream/7362128651158912501 {"id":7362128651158912501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/313e378f-dddd-4cf0-99b3-6da63e659233 {"id":"313e378f-dddd-4cf0-99b3-6da63e659233","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fdd158e 313e378f-dddd-4cf0-99b3-6da63e659233 /tidb/cdc/default/default/upstream/7362128651158912501 {"id":7362128651158912501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/313e378f-dddd-4cf0-99b3-6da63e659233 {"id":"313e378f-dddd-4cf0-99b3-6da63e659233","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fdd158e 313e378f-dddd-4cf0-99b3-6da63e659233 /tidb/cdc/default/default/upstream/7362128651158912501 {"id":7362128651158912501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.35327.out cli changefeed create --start-ts=449348687450341380 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 567802ad-785e-4062-920a-466a94512c33 Info: {"upstream_id":7362128651158912501,"namespace":"default","id":"567802ad-785e-4062-920a-466a94512c33","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:00:13.063041558+08:00","start_ts":449348687450341380,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348687450341380,"checkpoint_ts":449348687450341380,"checkpoint_time":"2024-04-26 19:00:09.329"} PASS Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd0cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22169, start at 2024-04-26 18:59:31.535595727 +0800 CST m=+4.245140118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.542 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.507 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683dd2500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:22262, start at 2024-04-26 18:59:31.649243706 +0800 CST m=+4.283901169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:31.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:31.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:31.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26169.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT + tso='449348688983621633 PASS bootstrapped True Bootstrap flag. Do not delete. coverage: 1.9% of statements in github.com/pingcap/tiflow/...' tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. + echo 449348688983621633 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... new_collation_enabled True If the new collations are enabled. Do not edit it. + awk -F ' ' '{print $1}' + set +x tikv_gc_leader_uuid 63c684067140015 Current GC worker leader UUID. (DO NOT EDIT) + start_ts=449348688983621633 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:00:15 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:37435, start at 2024-04-26 19:00:13.914098247 +0800 CST m=+4.424746578 Host name and pid of current GC leader. (DO NOT EDIT) + get_info_fail_msg='failed to get info:' tikv_gc_leader_lease 20240426-19:02:13.922 +0800 Current GC worker leader lease. (DO NOT EDIT) + etcd_info_msg='etcd info' + '[' -z '' ']' tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2620926211.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) tikv_gc_last_run_time 20240426-19:00:13.893 +0800 The time when last GC starts. (DO NOT EDIT) + (( i <= 50 )) tikv_gc_safe_point 20240426-18:50:13.893 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 2 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... table mark.finish_mark_3 not exists for 2-th check, retry later Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:18 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec00ece9-0410-4de9-aa64-e5b04ad20fd4 {"id":"ec00ece9-0410-4de9-aa64-e5b04ad20fd4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0ff5ec8e ec00ece9-0410-4de9-aa64-e5b04ad20fd4 /tidb/cdc/default/default/upstream/7362128700751312545 {"id":7362128700751312545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec00ece9-0410-4de9-aa64-e5b04ad20fd4 {"id":"ec00ece9-0410-4de9-aa64-e5b04ad20fd4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0ff5ec8e ec00ece9-0410-4de9-aa64-e5b04ad20fd4 /tidb/cdc/default/default/upstream/7362128700751312545 {"id":7362128700751312545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec00ece9-0410-4de9-aa64-e5b04ad20fd4 {"id":"ec00ece9-0410-4de9-aa64-e5b04ad20fd4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0ff5ec8e ec00ece9-0410-4de9-aa64-e5b04ad20fd4 /tidb/cdc/default/default/upstream/7362128700751312545 {"id":7362128700751312545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449348688983621633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26278.out cli changefeed create --start-ts=449348688983621633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Starting TLS TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684067140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:37435, start at 2024-04-26 19:00:13.914098247 +0800 CST m=+4.424746578 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:13.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:13.893 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:13.893 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 19:00:18.934 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/04/26 19:00:18.935 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] Starting CDC state checker... Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362128700751312545,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:00:18.856668006+08:00","start_ts":449348688983621633,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348688983621633,"checkpoint_ts":449348688983621633,"checkpoint_time":"2024-04-26 19:00:15.178"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4354 0 --:--:-- --:--:-- --:--:-- 4420 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:00:15.178","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:00:19.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:00:15.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:00:19.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:00:15.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:00:19.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:00:15.178' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:00:15.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:00:19.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:00:15.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:00:19.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:00:19' + echo 'sink_checkpoint_ts is 2024-04-26' 19:00:15.178 sink_checkpoint_ts is 2024-04-26 19:00:15.178 ++ date -d '2024-04-26 19:00:15.178' +%s + checkpoint_timestamp=1714129215 ++ date -d '2024-04-26 19:00:19' +%s + current_timestamp=1714129219 + '[' 4 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed The 3 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... check diff failed 2-th time, retry later Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 table mark.finish_mark_3 not exists for 4-th check, retry later Starting TLS TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c5600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27497, start at 2024-04-26 19:00:19.961766912 +0800 CST m=+4.616247403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log check diff failed 3-th time, retry later Starting TLS TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684067140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:37435, start at 2024-04-26 19:00:13.914098247 +0800 CST m=+4.424746578 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:13.922 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:13.893 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:13.893 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting CDC state checker... [2024/04/26 19:00:22.284 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDozMzA3LzAuMC4wLjA6MzAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:195\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [2024/04/26 19:00:22.284 +08:00] [FATAL] [terror.go:300] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] [stack="github.com/pingcap/tidb/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/parser/terror/terror.go:300\nmain.main\n\t/home/jenkins/agent/workspace/ld_tidb_multi_branch_release-6.5/go/src/github.com/pingcap/tidb/tidb-server/main.go:196\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250"] table mark.finish_mark_3 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tls tidb cluster failed [Fri Apr 26 19:00:23 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3923439236.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting CDC state checker... check diff failed 4-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 6-th check, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:26 GMT < Content-Length: 1048 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table mark.finish_mark_3 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Fri Apr 26 19:00:28 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.3929639298.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f97e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30853, start at 2024-04-26 19:00:00.668017981 +0800 CST m=+4.451515180 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:31 GMT < Content-Length: 1301 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc44b218-555f-4315-b0d1-bba983734109 {"id":"bc44b218-555f-4315-b0d1-bba983734109","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1d7 bc44b218-555f-4315-b0d1-bba983734109 /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc44b218-555f-4315-b0d1-bba983734109 {"id":"bc44b218-555f-4315-b0d1-bba983734109","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1d7 bc44b218-555f-4315-b0d1-bba983734109 /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc44b218-555f-4315-b0d1-bba983734109 {"id":"bc44b218-555f-4315-b0d1-bba983734109","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/bf0c077b-5646-49f6-a061-546d28183aea {"id":"bf0c077b-5646-49f6-a061-546d28183aea","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1b3 bf0c077b-5646-49f6-a061-546d28183aea /tidb/cdc/default/__cdc_meta__/owner/1f5d8f1a0ff3d1d7 bc44b218-555f-4315-b0d1-bba983734109 /tidb/cdc/default/default/upstream/7362128882126265983 {"id":7362128882126265983,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683ee8d80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25889, start at 2024-04-26 18:59:49.434984434 +0800 CST m=+4.237072929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.443 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683eeb280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--19xz8, pid:25988, start at 2024-04-26 18:59:49.614503933 +0800 CST m=+4.357896437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:01:49.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-18:59:49.578 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:49:49.578 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:00:32 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.3001530017.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 10-th check, retry later check_redo_resolved_ts 71c0cbf6-3ab7-412d-b285-28870fb34716 449348693344649220 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/71c0cbf6-3ab7-412d-b285-28870fb34716/meta global resolved ts 449348692820361219 not forward to 449348693344649220 run task failed 1-th time, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later table owner_resign.t1 exists check diff failed 3-th time, retry later check diff failed 1-th time, retry later pass test: check health pass test: get status check_redo_resolved_ts 71c0cbf6-3ab7-412d-b285-28870fb34716 449348693344649220 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/71c0cbf6-3ab7-412d-b285-28870fb34716/meta global resolved ts 449348693344649220 not forward to 449348693344649220 run task failed 2-th time, retry later pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s table mark.finish_mark_3 not exists for 11-th check, retry later + info='{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449348693942337537, "checkpoint_time": "2024-04-26 19:00:34.094", "error": null }' + echo '{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449348693942337537, "checkpoint_time": "2024-04-26 19:00:34.094", "error": null }' { "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449348693942337537, "checkpoint_time": "2024-04-26 19:00:34.094", "error": null } ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449348693942337537, '"checkpoint_time":' '"2024-04-26' '19:00:34.094",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449348693942337537, '"checkpoint_time":' '"2024-04-26' '19:00:34.094",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s + info='{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449348693968551942, "checkpoint_time": "2024-04-26 19:00:34.194", "error": null }' + echo '{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449348693968551942, "checkpoint_time": "2024-04-26 19:00:34.194", "error": null }' { "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449348693968551942, "checkpoint_time": "2024-04-26 19:00:34.194", "error": null } ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449348693968551942, '"checkpoint_time":' '"2024-04-26' '19:00:34.194",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449348693968551942, '"checkpoint_time":' '"2024-04-26' '19:00:34.194",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449348694007873544, "checkpoint_time": "2024-04-26 19:00:34.344", "error": null }' + echo '{ "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449348694007873544, "checkpoint_time": "2024-04-26 19:00:34.344", "error": null }' { "upstream_id": 7362128882126265983, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449348694007873544, "checkpoint_time": "2024-04-26 19:00:34.344", "error": null } ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449348694007873544, '"checkpoint_time":' '"2024-04-26' '19:00:34.344",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362128882126265983, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449348694007873544, '"checkpoint_time":' '"2024-04-26' '19:00:34.344",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:35 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71d76569-ba4f-4087-96f4-6911c55fff19 {"id":"71d76569-ba4f-4087-96f4-6911c55fff19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1037fa8e 71d76569-ba4f-4087-96f4-6911c55fff19 /tidb/cdc/default/default/upstream/7362128769726546574 {"id":7362128769726546574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71d76569-ba4f-4087-96f4-6911c55fff19 {"id":"71d76569-ba4f-4087-96f4-6911c55fff19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1037fa8e 71d76569-ba4f-4087-96f4-6911c55fff19 /tidb/cdc/default/default/upstream/7362128769726546574 {"id":7362128769726546574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/71d76569-ba4f-4087-96f4-6911c55fff19 {"id":"71d76569-ba4f-4087-96f4-6911c55fff19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1037fa8e 71d76569-ba4f-4087-96f4-6911c55fff19 /tidb/cdc/default/default/upstream/7362128769726546574 {"id":7362128769726546574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully [Fri Apr 26 19:00:35 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.2840228404.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later table mark.finish_mark_3 not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff failed 5-th time, retry later check_redo_resolved_ts 71c0cbf6-3ab7-412d-b285-28870fb34716 449348693344649220 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/71c0cbf6-3ab7-412d-b285-28870fb34716/meta run task successfully table mark.finish_mark_3 not exists for 13-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/04/26 19:00:38.710 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:00:38.711 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:00:38.711 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348694393225220] [resolvedTs=449348694393225220] [2024/04/26 19:00:38.711 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=357.12Β΅s] [2024/04/26 19:00:38.711 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=389.532Β΅s] [2024/04/26 19:00:38.712 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:00:38.712 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:00:38.717 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:00:38.719 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.720 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.721 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.721 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.721 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.721 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:00:38.721 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:00:38.721 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:00:38.721 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:00:38.728 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:00:38.730 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 19:00:38.730 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=449348694393225220] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:00:38.730 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully check diff successfully [Fri Apr 26 19:00:38 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/40969f45-4e63-4efb-a62c-2db38c1cf92a {ClusterID:default ID:default/40969f45-4e63-4efb-a62c-2db38c1cf92a Info:{"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} Status:0xc000a4cd98 TaskPositions:map[3783ac9a-cd8e-42a5-b96b-ce82241ded36:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3783ac9a-cd8e-42a5-b96b-ce82241ded36 {"id":"3783ac9a-cd8e-42a5-b96b-ce82241ded36","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f5359ba 3783ac9a-cd8e-42a5-b96b-ce82241ded36 /tidb/cdc/default/default/changefeed/info/40969f45-4e63-4efb-a62c-2db38c1cf92a {"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} /tidb/cdc/default/default/changefeed/status/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":449348695073488904,"min-table-barrier-ts":449348695073488904,"admin-job-type":0} /tidb/cdc/default/default/task/position/3783ac9a-cd8e-42a5-b96b-ce82241ded36/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/40969f45-4e63-4efb-a62c-2db38c1cf92a {ClusterID:default ID:default/40969f45-4e63-4efb-a62c-2db38c1cf92a Info:{"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} Status:0xc000a4cd98 TaskPositions:map[3783ac9a-cd8e-42a5-b96b-ce82241ded36:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3783ac9a-cd8e-42a5-b96b-ce82241ded36 {"id":"3783ac9a-cd8e-42a5-b96b-ce82241ded36","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f5359ba 3783ac9a-cd8e-42a5-b96b-ce82241ded36 /tidb/cdc/default/default/changefeed/info/40969f45-4e63-4efb-a62c-2db38c1cf92a {"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} /tidb/cdc/default/default/changefeed/status/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":449348695073488904,"min-table-barrier-ts":449348695073488904,"admin-job-type":0} /tidb/cdc/default/default/task/position/3783ac9a-cd8e-42a5-b96b-ce82241ded36/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/40969f45-4e63-4efb-a62c-2db38c1cf92a {ClusterID:default ID:default/40969f45-4e63-4efb-a62c-2db38c1cf92a Info:{"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} Status:0xc000a4cd98 TaskPositions:map[3783ac9a-cd8e-42a5-b96b-ce82241ded36:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3783ac9a-cd8e-42a5-b96b-ce82241ded36 {"id":"3783ac9a-cd8e-42a5-b96b-ce82241ded36","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 {"id":"66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f53588e 66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f5359ba 3783ac9a-cd8e-42a5-b96b-ce82241ded36 /tidb/cdc/default/default/changefeed/info/40969f45-4e63-4efb-a62c-2db38c1cf92a {"upstream-id":7362128517417682748,"namespace":"default","changefeed-id":"40969f45-4e63-4efb-a62c-2db38c1cf92a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T18:59:36.884149228+08:00","start-ts":449348678912311302,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-fi+ grep -q 'etcd info' le-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348678938525701} /tidb/cdc/default/default/changefeed/status/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":449348695073488904,"min-table-barrier-ts":449348695073488904,"admin-job-type":0} /tidb/cdc/default/default/task/position/3783ac9a-cd8e-42a5-b96b-ce82241ded36/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/66b97f94-c4d2-4ff8-83c8-2f22bfa4abc8/40969f45-4e63-4efb-a62c-2db38c1cf92a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128517417682748 {"id":7362128517417682748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 check diff successfully table mark.finish_mark_3 not exists for 14-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 1-th time, retry later check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 15-th check, retry later Starting Upstream TiDB... check diff failed 2-th time, retry later check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 16-th check, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table mark.finish_mark_3 not exists for 17-th check, retry later check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table processor_stop_delay.t not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) run test case test_kill_owner [Fri Apr 26 19:00:38 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4035540357.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:41 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.40420.out cli changefeed create --start-ts=449348679791804417 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 3990a444-9a6b-45c7-9bc3-bbb98798b335 Info: {"upstream_id":7362128520909889718,"namespace":"default","id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:00:41.451914742+08:00","start_ts":449348679791804417,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348679791804417,"checkpoint_ts":449348679791804417,"checkpoint_time":"2024-04-26 18:59:40.114"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 40360 owner id 1f2a3158-7b2b-4a61-807c-a9dae8cd494e [Fri Apr 26 19:00:42 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4053240534.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:45 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0001f72f0 TaskPositions:map[1f2a3158-7b2b-4a61-807c-a9dae8cd494e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d {"id":"fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d275d fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694707798022,"min-table-barrier-ts":449348694838870066,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2a3158-7b2b-4a61-807c-a9dae8cd494e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0001f72f0 TaskPositions:map[1f2a3158-7b2b-4a61-807c-a9dae8cd494e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d {"id":"fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d275d fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694707798022,"min-table-barrier-ts":449348694838870066,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2a3158-7b2b-4a61-807c-a9dae8cd494e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0001f72f0 TaskPositions:map[1f2a3158-7b2b-4a61-807c-a9dae8cd494e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2a3158-7b2b-4a61-807c-a9dae8cd494e {"id":"1f2a3158-7b2b-4a61-807c-a9dae8cd494e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d {"id":"fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2727 1f2a3158-7b2b-4a61-807c-a9dae8cd494e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d275d fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694707798022,"min-table-barrier-ts":449348694838870066,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2a3158-7b2b-4a61-807c-a9dae8cd494e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "1f2a3158-7b2b-4a61-807c-a9dae8cd494e" | grep -v cluster_id | grep id "cluster-id": "default" "id": "fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d", "cluster-id": "default" run task successfully capture_id: fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (40360) - No such process wait process 40360 exit for 1-th time... process 40360 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep fe8bf9b8-dcf3-4cdb-845c-5655374d4b7d -A1 | grep '"is-owner": true' "is-owner": true, run task successfully test_kill_owner: pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_owner [Fri Apr 26 19:00:47 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4071540717.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c5600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27497, start at 2024-04-26 19:00:19.961766912 +0800 CST m=+4.616247403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 4-th time, retry later table mark.finish_mark_3 not exists for 18-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table processor_stop_delay.t exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide check diff failed 1-th time, retry later * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9CF789BA3C085 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:00:50 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact check diff failed 5-th time, retry later Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002f29d28 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 80, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 84, resolvedTs: 449348694838870066, checkpointTs: 449348694838870066, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694838870066,"min-table-barrier-ts":449348698024181763,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002f29d28 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 80, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 84, resolvedTs: 449348694838870066, checkpointTs: 449348694838870066, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694838870066,"min-table-barrier-ts":449348698024181763,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002f29d28 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 80, resolvedTs: 449348698024181763, checkpointTs: 449348698024181763, state: Replicating tableID: 84, resolvedTs: 449348694838870066, checkpointTs: 449348694838870066, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348694838870066,"min-table-barrier-ts":449348698024181763,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 40720 owner id 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 [Fri Apr 26 19:00:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4086240864.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully [Fri Apr 26 19:00:40 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.3059730599.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:43 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 {"id":"ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f78eb14 ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 /tidb/cdc/default/default/upstream/7362128539668636333 {"id":7362128539668636333,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 {"id":"ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f78eb14 ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 /tidb/cdc/default/default/upstream/7362128539668636333 {"id":7362128539668636333,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 {"id":"ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f78eb14 ad8b1f1c-7bfe-4983-a1a0-dde7ce6ff703 /tidb/cdc/default/default/upstream/7362128539668636333 {"id":7362128539668636333,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.30655.out cli changefeed create --start-ts=449348680764882945 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 39b46855-43ff-4291-8b56-bf7bc7ff805a Info: {"upstream_id":7362128539668636333,"namespace":"default","id":"39b46855-43ff-4291-8b56-bf7bc7ff805a","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:00:44.23830458+08:00","start_ts":449348680764882945,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348680764882945,"checkpoint_ts":449348680764882945,"checkpoint_time":"2024-04-26 18:59:43.826"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:00:51 CST 2024] <<<<<< run test case simple success! >>>>>> Verifying downstream PD is started... check diff failed 6-th time, retry later table mark.finish_mark_3 not exists for 20-th check, retry later check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00327c2b8 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d8447527-ddf5-4fd2-8b8f-ef07916c7d4e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e {"id":"d8447527-ddf5-4fd2-8b8f-ef07916c7d4e","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27af d8447527-ddf5-4fd2-8b8f-ef07916c7d4e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348699072495621,"min-table-barrier-ts":449348699072495621,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00327c2b8 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d8447527-ddf5-4fd2-8b8f-ef07916c7d4e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e {"id":"d8447527-ddf5-4fd2-8b8f-ef07916c7d4e","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27af d8447527-ddf5-4fd2-8b8f-ef07916c7d4e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 + grep -q 'failed to get info:' {"checkpoint-ts":449348699072495621,"min-table-barrier-ts":449348699072495621,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00327c2b8 TaskPositions:map[1c1db9fe-7d42-41fd-9538-96c4b0f6a279:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} d8447527-ddf5-4fd2-8b8f-ef07916c7d4e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c1db9fe-7d42-41fd-9538-96c4b0f6a279 {"id":"1c1db9fe-7d42-41fd-9538-96c4b0f6a279","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e {"id":"d8447527-ddf5-4fd2-8b8f-ef07916c7d4e","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d278d 1c1db9fe-7d42-41fd-9538-96c4b0f6a279 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27af d8447527-ddf5-4fd2-8b8f-ef07916c7d4e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-fi+ grep -q 'etcd info' le-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348699072495621,"min-table-barrier-ts":449348699072495621,"admin-job-type":0} /tidb/cdc/default/default/task/position/1c1db9fe-7d42-41fd-9538-96c4b0f6a279/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d8447527-ddf5-4fd2-8b8f-ef07916c7d4e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "1c1db9fe-7d42-41fd-9538-96c4b0f6a279" | grep id "cluster-id": "default" "id": "d8447527-ddf5-4fd2-8b8f-ef07916c7d4e", "cluster-id": "default" run task successfully check diff successfully table mark.finish_mark_3 not exists for 21-th check, retry later capture_id: d8447527-ddf5-4fd2-8b8f-ef07916c7d4e ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'd8447527-ddf5-4fd2-8b8f-ef07916c7d4e' d8447527-ddf5-4fd2-8b8f-ef07916c7d4e run task successfully test_hang_up_owner: pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:00:54 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_owner [Fri Apr 26 19:00:55 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4100241004.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 22-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:00:57 CST 2024] <<<<<< run test case owner_resign success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f98a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30762, start at 2024-04-26 19:00:00.713563782 +0800 CST m=+4.563019279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c683f97e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:30853, start at 2024-04-26 19:00:00.668017981 +0800 CST m=+4.451515180 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:00.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:00.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:00.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:00:57 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3484334845.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 23-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:58 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000852558 TaskPositions:map[f230f011-4aa5-4d2b-b769-2ab2e8f2da7a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a {"id":"f230f011-4aa5-4d2b-b769-2ab2e8f2da7a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27d4 f230f011-4aa5-4d2b-b769-2ab2e8f2da7a /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348700121333764,"admin-job-type":0} /tidb/cdc/default/default/task/position/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000852558 TaskPositions:map[f230f011-4aa5-4d2b-b769-2ab2e8f2da7a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a {"id":"f230f011-4aa5-4d2b-b769-2ab2e8f2da7a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27d4 f230f011-4aa5-4d2b-b769-2ab2e8f2da7a /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348700121333764,"admin-job-type":0} /tidb/cdc/default/default/task/position/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000852558 TaskPositions:map[f230f011-4aa5-4d2b-b769-2ab2e8f2da7a:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a {"id":"f230f011-4aa5-4d2b-b769-2ab2e8f2da7a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27d4 f230f011-4aa5-4d2b-b769-2ab2e8f2da7a /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348700121333764,"admin-job-type":0} /tidb/cdc/default/default/task/position/f230f011-4aa5-4d2b-b769-2ab2e8f2da7a/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully owner pid: 41007 owner id f230f011-4aa5-4d2b-b769-2ab2e8f2da7a process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'f230f011-4aa5-4d2b-b769-2ab2e8f2da7a' /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27d4 run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully test_expire_owner pass check diff failed 4-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Fri Apr 26 19:01:00 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4119041192.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 23741 exit for 1-th time... wait process 23741 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (23741) - No such process wait process 23741 exit for 3-th time... process 23741 already exit [Fri Apr 26 19:00:50 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2391923921.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:00:57 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.2398223984.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:00:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {ClusterID:default ID:default/ddl-manager Info:{"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} Status:0xc003d5b0b0 TaskPositions:map[5abf4aa2-66c2-4e10-91e8-465f215dbf47:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5abf4aa2-66c2-4e10-91e8-465f215dbf47 {"id":"5abf4aa2-66c2-4e10-91e8-465f215dbf47","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f9189d5 5abf4aa2-66c2-4e10-91e8-465f215dbf47 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362128587817658600,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T18:59:52.75649859+08:00","start-ts":449348683071750149,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348683098226693} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348697162252321,"min-table-barrier-ts":449348697162252321,"admin-job-type":0} /tidb/cdc/default/default/task/position/5abf4aa2-66c2-4e10-91e8-465f215dbf47/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128587817658600 {"id":7362128587817658600,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 24-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:00 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/744693ae-07dd-48a8-83c2-1322d7cc0979 {"id":"744693ae-07dd-48a8-83c2-1322d7cc0979","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998690 744693ae-07dd-48a8-83c2-1322d7cc0979 /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/744693ae-07dd-48a8-83c2-1322d7cc0979 {"id":"744693ae-07dd-48a8-83c2-1322d7cc0979","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998690 744693ae-07dd-48a8-83c2-1322d7cc0979 /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/744693ae-07dd-48a8-83c2-1322d7cc0979 {"id":"744693ae-07dd-48a8-83c2-1322d7cc0979","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998690 744693ae-07dd-48a8-83c2-1322d7cc0979 /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 3-th check, retry later check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 25-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 4-th check, retry later check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:03 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002c114d0 TaskPositions:map[b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348701418684419,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002c114d0 TaskPositions:map[b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348701418684419,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002c114d0 TaskPositions:map[b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 84, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing tableID: 80, resolvedTs: 449348700121333764, checkpointTs: 449348700121333764, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348700121333764,"min-table-barrier-ts":449348701418684419,"admin-job-type":0} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 41195 owner id b9924f43-16f6-4efa-9e7d-58f60f58d945 [Fri Apr 26 19:01:03 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4132941331.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684367d80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33199, start at 2024-04-26 19:01:03.139439462 +0800 CST m=+4.439254542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log table ddl_manager.finish_mark not exists for 5-th check, retry later check diff successfully + workdir=/tmp/tidb_cdc_test/canal_json_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_basic ++ date + echo '[Fri Apr 26 19:01:03 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>>' [Fri Apr 26 19:01:03 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml table mark.finish_mark_3 not exists for 26-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:01:05 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Starting CDC state checker... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001904720 TaskPositions:map[ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702218485768,"min-table-barrier-ts":449348702467260423,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001904720 TaskPositions:map[ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702218485768,"min-table-barrier-ts":449348702467260423,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001904720 TaskPositions:map[ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702218485768,"min-table-barrier-ts":449348702467260423,"admin-job-type":0} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "b9924f43-16f6-4efa-9e7d-58f60f58d945" | grep id Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "cluster-id": "default" "id": "ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335", "cluster-id": "default" run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.simple1 exists pass test: list changefeed pass test: get changefeed pass test: pause changefeed pass test: update changefeed pass test: resume changefeed pass test: rebalance table pass test: list processors table mark.finish_mark_3 not exists for 27-th check, retry later pass test: get processors table ddl_manager.finish_mark not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release pass test: move table pass test: set log level Verifying downstream PD is started... capture_id: ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... pass test: remove changefeed start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... pass test: resign owner pass test: get tso Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... pass test: verify table table ddl_manager.finish_mark not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: create changefeed v2 table mark.finish_mark_3 not exists for 28-th check, retry later status code 200 pass test: show metadata status code 204 status code 204 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:01:10 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c3240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27419, start at 2024-04-26 19:00:19.791783293 +0800 CST m=+4.518016168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.798 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6840c5600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:27497, start at 2024-04-26 19:00:19.961766912 +0800 CST m=+4.616247403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:02:19.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:00:19.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:50:19.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:01:11 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.3152631528.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 [Fri Apr 26 19:01:11 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4148141483.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 29-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "mysql.db"="move_table" "workload"="core" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "operationcount"="0" "mysql.port"="4000" "readproportion"="0" "mysql.user"="root" "scanproportion"="0" "recordcount"="10000" "updateproportion"="0" "dotransactions"="false" "threadcount"="10" ********************************************** Run finished, takes 2.015042288s INSERT - Takes(s): 2.0, Count: 10000, OPS: 4976.2, Avg(us): 1967, Min(us): 789, Max(us): 41605, 95th(us): 3000, 99th(us): 36000 [Fri Apr 26 19:01:04 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2423724239.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:07 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24291.out cli changefeed create --start-ts=449348686198079489 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info: {"upstream_id":7362128673217335552,"namespace":"default","id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:01:07.955497741+08:00","start_ts":449348686198079489,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348686198079489,"checkpoint_ts":449348686198079489,"checkpoint_time":"2024-04-26 19:00:04.552"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 26 19:01:08 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2432524327.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc000b62420 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701363109911,"min-table-barrier-ts":449348703486738440,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc000b62420 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701363109911,"min-table-barrier-ts":449348703486738440,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc000b62420 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701363109911,"min-table-barrier-ts":449348703486738440,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:01:11 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2438524387.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 30-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc0017127b0 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e701113f-c0ca-4ff9-a3d3-92b150f3b240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e701113f-c0ca-4ff9-a3d3-92b150f3b240 {"id":"e701113f-c0ca-4ff9-a3d3-92b150f3b240","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b67 e701113f-c0ca-4ff9-a3d3-92b150f3b240 /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701507289175,"min-table-barrier-ts":449348704278937606,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e701113f-c0ca-4ff9-a3d3-92b150f3b240/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc0017127b0 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e701113f-c0ca-4ff9-a3d3-92b150f3b240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e701113f-c0ca-4ff9-a3d3-92b150f3b240 {"id":"e701113f-c0ca-4ff9-a3d3-92b150f3b240","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b67 e701113f-c0ca-4ff9-a3d3-92b150f3b240 /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701507289175,"min-table-barrier-ts":449348704278937606,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e701113f-c0ca-4ff9-a3d3-92b150f3b240/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {ClusterID:default ID:default/8c69778e-ee28-4cbc-9384-f2b6e55d5818 Info:{"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} Status:0xc0017127b0 TaskPositions:map[21b0e157-ee4e-499b-a090-997527edf59b:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 62f8d21f-9d8d-4594-92f3-643b638908cb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} e701113f-c0ca-4ff9-a3d3-92b150f3b240:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21b0e157-ee4e-499b-a090-997527edf59b {"id":"21b0e157-ee4e-499b-a090-997527edf59b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/62f8d21f-9d8d-4594-92f3-643b638908cb {"id":"62f8d21f-9d8d-4594-92f3-643b638908cb","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e701113f-c0ca-4ff9-a3d3-92b150f3b240 {"id":"e701113f-c0ca-4ff9-a3d3-92b150f3b240","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b19 21b0e157-ee4e-499b-a090-997527edf59b /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b4f 62f8d21f-9d8d-4594-92f3-643b638908cb /tidb/cdc/default/__cdc_meta__/owner/22318f1a0fd43b67 e701113f-c0ca-4ff9-a3d3-92b150f3b240 /tidb/cdc/default/default/changefeed/info/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"upstream-id":7362128673217335552,"namespace":"default","changefeed-id":"8c69778e-ee28-4cbc-9384-f2b6e55d5818","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:01:07.955497741+08:00","start-ts":449348686198079489,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348702818271233} /tidb/cdc/default/default/changefeed/status/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":449348701507289175,"min-table-barrier-ts":449348704278937606,"admin-job-type":0} /tidb/cdc/default/default/task/position/21b0e157-ee4e-499b-a090-997527edf59b/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/62f8d21f-9d8d-4594-92f3-643b638908cb/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e701113f-c0ca-4ff9-a3d3-92b150f3b240/8c69778e-ee28-4cbc-9384-f2b6e55d5818 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128673217335552 {"id":7362128673217335552,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001374f60 TaskPositions:map[21aa91be-e245-4326-8f2a-afd7bb8d98d3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21aa91be-e245-4326-8f2a-afd7bb8d98d3 {"id":"21aa91be-e245-4326-8f2a-afd7bb8d98d3","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2863 21aa91be-e245-4326-8f2a-afd7bb8d98d3 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702729666567,"min-table-barrier-ts":449348702729666567,"admin-job-type":0} /tidb/cdc/default/default/task/position/21aa91be-e245-4326-8f2a-afd7bb8d98d3/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001374f60 TaskPositions:map[21aa91be-e245-4326-8f2a-afd7bb8d98d3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21aa91be-e245-4326-8f2a-afd7bb8d98d3 {"id":"21aa91be-e245-4326-8f2a-afd7bb8d98d3","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2863 21aa91be-e245-4326-8f2a-afd7bb8d98d3 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-com+ grep -q 'failed to get info:' mit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702729666567,"min-table-barrier-ts":449348702729666567,"admin-job-type":0} /tidb/cdc/default/default/task/position/21aa91be-e245-4326-8f2a-afd7bb8d98d3/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc001374f60 TaskPositions:map[21aa91be-e245-4326-8f2a-afd7bb8d98d3:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b9924f43-16f6-4efa-9e7d-58f60f58d945:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/21aa91be-e245-4326-8f2a-afd7bb8d98d3 {"id":"21aa91be-e245-4326-8f2a-afd7bb8d98d3","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 {"id":"ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b9924f43-16f6-4efa-9e7d-58f60f58d945 {"id":"b9924f43-16f6-4efa-9e7d-58f60f58d945","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d27fe b9924f43-16f6-4efa-9e7d-58f60f58d945 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2822 ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2863 21aa91be-e245-4326-8f2a-afd7bb8d98d3 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348702729666567,"min-table-barrier-ts":449348702729666567,"admin-job-type":0} /tidb/cdc/default/default/task/position/21aa91be-e245-4326-8f2a-afd7bb8d98d3/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ae0f73c9-70c8-4e5f-8cbd-8eceffeb7335/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b9924f43-16f6-4efa-9e7d-58f60f58d945/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' table move_table.usertable exists run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:14 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18022a7c-ee98-49ea-8948-9ee61e85ddf8 {"id":"18022a7c-ee98-49ea-8948-9ee61e85ddf8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10d42a90 18022a7c-ee98-49ea-8948-9ee61e85ddf8 /tidb/cdc/default/default/upstream/7362128910180436817 {"id":7362128910180436817,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18022a7c-ee98-49ea-8948-9ee61e85ddf8 {"id":"18022a7c-ee98-49ea-8948-9ee61e85ddf8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10d42a90 18022a7c-ee98-49ea-8948-9ee61e85ddf8 /tidb/cdc/default/default/upstream/7362128910180436817 {"id":7362128910180436817,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18022a7c-ee98-49ea-8948-9ee61e85ddf8 {"id":"18022a7c-ee98-49ea-8948-9ee61e85ddf8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10d42a90 18022a7c-ee98-49ea-8948-9ee61e85ddf8 /tidb/cdc/default/default/upstream/7362128910180436817 {"id":7362128910180436817,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7362128910180436817,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:01:14.62845538+08:00","start_ts":449348703715065857,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348703715065857,"checkpoint_ts":449348703715065857,"checkpoint_time":"2024-04-26 19:01:11.374"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7362128910180436817,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:01:14.828375501+08:00","start_ts":449348703715065857,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348703715065857,"checkpoint_ts":449348703715065857,"checkpoint_time":"2024-04-26 19:01:11.374"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7362128910180436817,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:01:15.030660434+08:00","start_ts":449348703715065857,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348703715065857,"checkpoint_ts":449348703715065857,"checkpoint_time":"2024-04-26 19:01:11.374"} table ddl_manager.finish_mark not exists for 10-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 check diff successfully [Fri Apr 26 19:01:15 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> table mark.finish_mark_3 not exists for 31-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684414600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31744, start at 2024-04-26 19:01:14.157595892 +0800 CST m=+4.453959411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 2-th time, retry later table mark.finish_mark_3 not exists for 32-th check, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443da40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35692, start at 2024-04-26 19:01:16.80678516 +0800 CST m=+4.619540514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 12-th check, retry later table mark.finish_mark_3 not exists for 33-th check, retry later Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b440016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29709, start at 2024-04-26 19:01:18.706820322 +0800 CST m=+4.393321551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2388 0 --:--:-- --:--:-- --:--:-- 2405 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:01:19.029","puller_resolved_ts":"2024-04-26 19:01:20.029","last_synced_ts":"2024-04-26 19:01:13.029","now_ts":"2024-04-26 19:01:20.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:01:19.029","puller_resolved_ts":"2024-04-26' '19:01:20.029","last_synced_ts":"2024-04-26' '19:01:13.029","now_ts":"2024-04-26' '19:01:20.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:01:19.029","puller_resolved_ts":"2024-04-26' '19:01:20.029","last_synced_ts":"2024-04-26' '19:01:13.029","now_ts":"2024-04-26' '19:01:20.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.68.177 + export UP_TIDB_HOST=10.233.68.177 UP_PD_HOST_1=10.233.68.177 UP_PD_HOST_2=10.233.68.177 UP_PD_HOST_3=10.233.68.177 UP_TIKV_HOST_1=10.233.68.177 UP_TIKV_HOST_2=10.233.68.177 UP_TIKV_HOST_3=10.233.68.177 + UP_TIDB_HOST=10.233.68.177 + UP_PD_HOST_1=10.233.68.177 + UP_PD_HOST_2=10.233.68.177 + UP_PD_HOST_3=10.233.68.177 + UP_TIKV_HOST_1=10.233.68.177 + UP_TIKV_HOST_2=10.233.68.177 + UP_TIKV_HOST_3=10.233.68.177 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=17294 + trap 'stop_tidb_cluster && stop_proxy' EXIT + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.68.177' UP_TIDB_HOST 10.233.68.177 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.68.177' UP_PD_HOST_1 10.233.68.177 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.68.177' UP_PD_HOST_2 10.233.68.177 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.68.177' UP_PD_HOST_3 10.233.68.177 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.68.177' UP_TIKV_HOST_1 10.233.68.177 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.68.177' UP_TIKV_HOST_2 10.233.68.177 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.68.177' UP_TIKV_HOST_3 10.233.68.177 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 13-th check, retry later table mark.finish_mark_3 not exists for 34-th check, retry later Starting CDC state checker... + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... [2024/04/26 19:01:22.780 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:01:22.782 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:01:22.783 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=21b0e157-ee4e-499b-a090-997527edf59b] [2024/04/26 19:01:22.783 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:01:22.784 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:01:22.933 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=21b0e157-ee4e-499b-a090-997527edf59b] [processorDetail="{\"table_ids\":[82]}"] [2024/04/26 19:01:23.083 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=62f8d21f-9d8d-4594-92f3-643b638908cb] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:01:23.233 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=e701113f-c0ca-4ff9-a3d3-92b150f3b240] [processorDetail="{\"table_ids\":[84]}"] [2024/04/26 19:01:23.233 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"21b0e157-ee4e-499b-a090-997527edf59b\":[{\"ID\":82,\"Changefeed\":\"8c69778e-ee28-4cbc-9384-f2b6e55d5818\"}],\"62f8d21f-9d8d-4594-92f3-643b638908cb\":[],\"e701113f-c0ca-4ff9-a3d3-92b150f3b240\":[{\"ID\":84,\"Changefeed\":\"8c69778e-ee28-4cbc-9384-f2b6e55d5818\"}]}"] [2024/04/26 19:01:23.233 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=8c69778e-ee28-4cbc-9384-f2b6e55d5818&target-cp-id=62f8d21f-9d8d-4594-92f3-643b638908cb&table-id=82"] [2024/04/26 19:01:23.283 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=82] [2024/04/26 19:01:23.283 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=21b0e157-ee4e-499b-a090-997527edf59b] [targetCapture=62f8d21f-9d8d-4594-92f3-643b638908cb] table move_table.check1 exists check diff successfully table ddl_manager.finish_mark not exists for 14-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 not exists for 35-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... table ddl_manager.finish_mark not exists for 15-th check, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 36-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684367d80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33199, start at 2024-04-26 19:01:03.139439462 +0800 CST m=+4.439254542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass table mark.finish_mark_3 not exists for 37-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 16-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_owner_retryable_error [Fri Apr 26 19:01:28 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4174741749.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:01:28.038 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:01:28.040 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:01:28.041 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=21b0e157-ee4e-499b-a090-997527edf59b] [2024/04/26 19:01:28.041 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:01:28.041 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/04/26 19:01:28.183 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=21b0e157-ee4e-499b-a090-997527edf59b] [processorDetail="{\"table_ids\":[86]}"] [2024/04/26 19:01:28.332 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=62f8d21f-9d8d-4594-92f3-643b638908cb] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:01:28.483 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=8c69778e-ee28-4cbc-9384-f2b6e55d5818] [captureID=e701113f-c0ca-4ff9-a3d3-92b150f3b240] [processorDetail="{\"table_ids\":[84]}"] [2024/04/26 19:01:28.483 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"21b0e157-ee4e-499b-a090-997527edf59b\":[{\"ID\":86,\"Changefeed\":\"8c69778e-ee28-4cbc-9384-f2b6e55d5818\"}],\"62f8d21f-9d8d-4594-92f3-643b638908cb\":[],\"e701113f-c0ca-4ff9-a3d3-92b150f3b240\":[{\"ID\":84,\"Changefeed\":\"8c69778e-ee28-4cbc-9384-f2b6e55d5818\"}]}"] [2024/04/26 19:01:28.483 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=8c69778e-ee28-4cbc-9384-f2b6e55d5818&target-cp-id=e701113f-c0ca-4ff9-a3d3-92b150f3b240&table-id=86"] [2024/04/26 19:01:28.533 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=86] [2024/04/26 19:01:28.533 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=21b0e157-ee4e-499b-a090-997527edf59b] [targetCapture=e701113f-c0ca-4ff9-a3d3-92b150f3b240] check diff successfully table move_table.check2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... table ddl_manager.finish_mark not exists for 17-th check, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.68.177' (111) table mark.finish_mark_3 not exists for 38-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f7580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27818, start at 2024-04-26 19:01:28.698771856 +0800 CST m=+4.438861316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.705 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log table move_table.check2 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000aa3488 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 84, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 80, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348707710140422,"min-table-barrier-ts":449348708758978565,"admin-job-type":0} /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000aa3488 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 84, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 80, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348707710140422,"min-table-barrier-ts":449348708758978565,"admin-job-type":0} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000aa3488 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 84, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing tableID: 80, resolvedTs: 449348707710140422, checkpointTs: 449348707710140422, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348707710140422,"min-table-barrier-ts":449348708758978565,"admin-job-type":0} /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... table ddl_manager.finish_mark not exists for 18-th check, retry later Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.68.177' (111) owner pid: 41752 owner id 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 [Fri Apr 26 19:01:31 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4188441886.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 39-th check, retry later Starting CDC state checker... table move_table.check2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:01:33 CST 2024] <<<<<< run test case move_table success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 19-th check, retry later table mark.finish_mark_3 not exists for 40-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.68.177' (111) [2024/04/26 19:01:28.115 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 21"] [2024/04/26 19:01:33.118 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 29"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0007c9590 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b09cca24-2949-4420-8685-fa793749c697:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b09cca24-2949-4420-8685-fa793749c697 {"id":"b09cca24-2949-4420-8685-fa793749c697","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28e3 b09cca24-2949-4420-8685-fa793749c697 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709558255618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b09cca24-2949-4420-8685-fa793749c697/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0007c9590 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b09cca24-2949-4420-8685-fa793749c697:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b09cca24-2949-4420-8685-fa793749c697 {"id":"b09cca24-2949-4420-8685-fa793749c697","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28e3 b09cca24-2949-4420-8685-fa793749c697 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709558255618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b09cca24-2949-4420-8685-fa793749c697/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0007c9590 TaskPositions:map[53af3d2b-b4ac-4db8-ac83-cc39bf0537b7:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b09cca24-2949-4420-8685-fa793749c697:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 {"id":"53af3d2b-b4ac-4db8-ac83-cc39bf0537b7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b09cca24-2949-4420-8685-fa793749c697 {"id":"b09cca24-2949-4420-8685-fa793749c697","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28c1 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d28e3 b09cca24-2949-4420-8685-fa793749c697 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709558255618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/53af3d2b-b4ac-4db8-ac83-cc39bf0537b7/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b09cca24-2949-4420-8685-fa793749c697/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "53af3d2b-b4ac-4db8-ac83-cc39bf0537b7" | grep id "cluster-id": "default" "id": "b09cca24-2949-4420-8685-fa793749c697", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 20-th check, retry later capture_id: b09cca24-2949-4420-8685-fa793749c697 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 10583 0 --:--:-- --:--:-- --:--:-- 11666 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '53af3d2b-b4ac-4db8-ac83-cc39bf0537b7' 53af3d2b-b4ac-4db8-ac83-cc39bf0537b7 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 41-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Fri Apr 26 19:01:36 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4204242044.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454c000019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40830, start at 2024-04-26 19:01:34.122908496 +0800 CST m=+4.183548509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684414600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31744, start at 2024-04-26 19:01:14.157595892 +0800 CST m=+4.453959411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 21-th check, retry later Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 42-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table ddl_manager.finish_mark not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 43-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443da40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35692, start at 2024-04-26 19:01:16.80678516 +0800 CST m=+4.619540514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 44-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00209aba0 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709820399618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00209aba0 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SU+ grep -q 'failed to get info:' BSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709820399618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00209aba0 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348709820399618,"min-table-barrier-ts":449348709820399618,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b1e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31427, start at 2024-04-26 19:01:40.633078277 +0800 CST m=+4.326269964 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/error.log "is-owner": true, run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 19:01:42 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... owner pid: 42047 owner id 4073378f-339f-43c7-b443-0064b15f5c91 [Fri Apr 26 19:01:43 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4217142173.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b440016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29709, start at 2024-04-26 19:01:18.706820322 +0800 CST m=+4.393321551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 24-th check, retry later table mark.finish_mark_3 not exists for 45-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... table ddl_manager.finish_mark not exists for 25-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 46-th check, retry later Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc003a0cc18 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 40ccb52a-ca60-47e8-b977-41bc832e30a0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/40ccb52a-ca60-47e8-b977-41bc832e30a0 {"id":"40ccb52a-ca60-47e8-b977-41bc832e30a0","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2937 40ccb52a-ca60-47e8-b977-41bc832e30a0 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348712703983619,"min-table-barrier-ts":449348712703983620,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/40ccb52a-ca60-47e8-b977-41bc832e30a0/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc003a0cc18 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 40ccb52a-ca60-47e8-b977-41bc832e30a0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/40ccb52a-ca60-47e8-b977-41bc832e30a0 {"id":"40ccb52a-ca60-47e8-b977-41bc832e30a0","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2937 40ccb52a-ca60-47e8-b977-41bc832e30a0 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":20+ grep -q 'failed to get info:' 0,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348712703983619,"min-table-barrier-ts":449348712703983620,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/40ccb52a-ca60-47e8-b977-41bc832e30a0/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc003a0cc18 TaskPositions:map[4073378f-339f-43c7-b443-0064b15f5c91:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 40ccb52a-ca60-47e8-b977-41bc832e30a0:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4073378f-339f-43c7-b443-0064b15f5c91 {"id":"4073378f-339f-43c7-b443-0064b15f5c91","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/40ccb52a-ca60-47e8-b977-41bc832e30a0 {"id":"40ccb52a-ca60-47e8-b977-41bc832e30a0","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d290e 4073378f-339f-43c7-b443-0064b15f5c91 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2937 40ccb52a-ca60-47e8-b977-41bc832e30a0 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":20+ grep -q 'etcd info' 0,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348712703983619,"min-table-barrier-ts":449348712703983620,"admin-job-type":0} /tidb/cdc/default/default/task/position/4073378f-339f-43c7-b443-0064b15f5c91/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/40ccb52a-ca60-47e8-b977-41bc832e30a0/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "4073378f-339f-43c7-b443-0064b15f5c91" | grep id "cluster-id": "default" "id": "40ccb52a-ca60-47e8-b977-41bc832e30a0", "cluster-id": "default" run task successfully capture_id: 40ccb52a-ca60-47e8-b977-41bc832e30a0 start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 26-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684368980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33098, start at 2024-04-26 19:01:03.175319171 +0800 CST m=+4.535374675 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684367d80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:33199, start at 2024-04-26 19:01:03.139439462 +0800 CST m=+4.439254542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:03.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:03.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:03.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:01:47 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3725537257.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 47-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 27-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 48-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Verifying downstream PD is started... < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:01:50 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 {"id":"f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e928e f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 {"id":"f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e928e f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 {"id":"f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e928e f35ed86b-0ed9-4cb4-88d5-5cc8c9304cc1 /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 28-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 49-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f7580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27818, start at 2024-04-26 19:01:28.698771856 +0800 CST m=+4.438861316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.705 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 29-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 50-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 30-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 51-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.68.177' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table2.t2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684412ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31643, start at 2024-04-26 19:01:14.086299753 +0800 CST m=+4.449135212 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.094 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.093 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.093 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684414600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:31744, start at 2024-04-26 19:01:14.157595892 +0800 CST m=+4.453959411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:14.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:14.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:14.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:01:57 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.3588335885.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 31-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 52-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454c000019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40830, start at 2024-04-26 19:01:34.122908496 +0800 CST m=+4.183548509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table2.t2 not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 32-th check, retry later table mark.finish_mark_3 not exists for 53-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846ca880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26504, start at 2024-04-26 19:01:58.608529813 +0800 CST m=+4.366270859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.611 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.611 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:00 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/390bf7a6-85d3-4440-aa7d-78ecbab5ffec {"id":"390bf7a6-85d3-4440-aa7d-78ecbab5ffec","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1185188e 390bf7a6-85d3-4440-aa7d-78ecbab5ffec /tidb/cdc/default/default/upstream/7362129138965325584 {"id":7362129138965325584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/390bf7a6-85d3-4440-aa7d-78ecbab5ffec {"id":"390bf7a6-85d3-4440-aa7d-78ecbab5ffec","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1185188e 390bf7a6-85d3-4440-aa7d-78ecbab5ffec /tidb/cdc/default/default/upstream/7362129138965325584 {"id":7362129138965325584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/390bf7a6-85d3-4440-aa7d-78ecbab5ffec {"id":"390bf7a6-85d3-4440-aa7d-78ecbab5ffec","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1185188e 390bf7a6-85d3-4440-aa7d-78ecbab5ffec /tidb/cdc/default/default/upstream/7362129138965325584 {"id":7362129138965325584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443f2c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35600, start at 2024-04-26 19:01:16.882235775 +0800 CST m=+4.772656209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68443da40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:35692, start at 2024-04-26 19:01:16.80678516 +0800 CST m=+4.619540514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:16.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:16.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:16.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:02:01 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.3980439806.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table2.t2 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later table mark.finish_mark_3 not exists for 54-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b50000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29612, start at 2024-04-26 19:01:18.691924565 +0800 CST m=+4.434273899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.676 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.676 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68445b440016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:29709, start at 2024-04-26 19:01:18.706820322 +0800 CST m=+4.393321551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:18.713 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:18.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:18.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.33772.out cli tso query --pd=http://127.0.0.1:2379 empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348717166985217 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348717166985217 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:02:03 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.3382233824.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table2.t2 not exists for 5-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 34-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:04 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11efef18-3ac7-46af-9d28-32a821b00d66 {"id":"11efef18-3ac7-46af-9d28-32a821b00d66","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192ef91 11efef18-3ac7-46af-9d28-32a821b00d66 /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11efef18-3ac7-46af-9d28-32a821b00d66 {"id":"11efef18-3ac7-46af-9d28-32a821b00d66","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192ef91 11efef18-3ac7-46af-9d28-32a821b00d66 /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11efef18-3ac7-46af-9d28-32a821b00d66 {"id":"11efef18-3ac7-46af-9d28-32a821b00d66","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192ef91 11efef18-3ac7-46af-9d28-32a821b00d66 /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 55-th check, retry later table partition_table2.t2 not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b1e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31427, start at 2024-04-26 19:01:40.633078277 +0800 CST m=+4.326269964 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:06 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc008543-521d-49be-8128-9d3e97d8d789 {"id":"bc008543-521d-49be-8128-9d3e97d8d789","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1199af8a bc008543-521d-49be-8128-9d3e97d8d789 /tidb/cdc/default/default/upstream/7362129156768881038 {"id":7362129156768881038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc008543-521d-49be-8128-9d3e97d8d789 {"id":"bc008543-521d-49be-8128-9d3e97d8d789","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1199af8a bc008543-521d-49be-8128-9d3e97d8d789 /tidb/cdc/default/default/upstream/7362129156768881038 {"id":7362129156768881038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc008543-521d-49be-8128-9d3e97d8d789 {"id":"bc008543-521d-49be-8128-9d3e97d8d789","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1199af8a bc008543-521d-49be-8128-9d3e97d8d789 /tidb/cdc/default/default/upstream/7362129156768881038 {"id":7362129156768881038,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.33888.out cli changefeed create --start-ts=449348717166985217 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0076b9b5-f25b-4c1b-b746-11ccf8d2878b Info: {"upstream_id":7362129156768881038,"namespace":"default","id":"0076b9b5-f25b-4c1b-b746-11ccf8d2878b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:02:06.43498053+08:00","start_ts":449348717166985217,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348717166985217,"checkpoint_ts":449348717166985217,"checkpoint_time":"2024-04-26 19:02:02.689"} PASS nonempty select id, val from test.availability3 where id=1 and val=22 table mark.finish_mark_3 not exists for 56-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... table partition_table2.t2 not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later table mark.finish_mark_3 not exists for 57-th check, retry later empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case delete_owner_key [Fri Apr 26 19:02:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4259142593.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table2.t2 not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later table mark.finish_mark_3 not exists for 58-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table2.t2 not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 38-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:02:03 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 19:02:11 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> table mark.finish_mark_3 exists table mark.finish_mark exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f9d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27735, start at 2024-04-26 19:01:28.862613024 +0800 CST m=+4.682164249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6844f7580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:27818, start at 2024-04-26 19:01:28.698771856 +0800 CST m=+4.438861316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:28.705 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:28.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:28.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:02:12 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.3182531827.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:02:13 CST 2024] <<<<<< run test case default_value success! >>>>>> table ddl_manager.finish_mark not exists for 39-th check, retry later table partition_table2.t2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000eb8720 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating tableID: 82, resolvedTs: 449348720155164674, checkpointTs: 449348719545942021, state: Replicating tableID: 84, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348719545942021,"min-table-barrier-ts":449348720155164674,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000eb8720 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating tableID: 82, resolvedTs: 449348720155164674, checkpointTs: 449348719545942021, state: Replicating tableID: 84, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348719545942021,"min-table-barrier-ts":449348720155164674,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000eb8720 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating tableID: 82, resolvedTs: 449348720155164674, checkpointTs: 449348719545942021, state: Replicating tableID: 84, resolvedTs: 449348720155164675, checkpointTs: 449348719545942021, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348719545942021,"min-table-barrier-ts":449348720155164674,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 42596 owner id 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e default owner key /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 [Fri Apr 26 19:02:14 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4274742749.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 40-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:15 GMT < Content-Length: 656 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619f1db8-4b3f-4028-902d-818c1770665e {"id":"619f1db8-4b3f-4028-902d-818c1770665e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11c35561 619f1db8-4b3f-4028-902d-818c1770665e /tidb/cdc/default/default/upstream/7362129205688604073 {"id":7362129205688604073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619f1db8-4b3f-4028-902d-818c1770665e {"id":"619f1db8-4b3f-4028-902d-818c1770665e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11c35561 619f1db8-4b3f-4028-902d-818c1770665e /tidb/cdc/default/default/upstream/7362129205688604073 {"id":7362129205688604073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/619f1db8-4b3f-4028-902d-818c1770665e {"id":"619f1db8-4b3f-4028-902d-818c1770665e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11c35561 619f1db8-4b3f-4028-902d-818c1770665e /tidb/cdc/default/default/upstream/7362129205688604073 {"id":7362129205688604073,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.31880.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } Create changefeed successfully! ID: 59b77de3-3f85-4297-8549-3d0a62455bc2 Info: {"upstream_id":7362129205688604073,"namespace":"default","id":"59b77de3-3f85-4297-8549-3d0a62455bc2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:02:16.075903566+08:00","start_ts":449348720643538948,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348720643538948,"checkpoint_ts":449348720643538948,"checkpoint_time":"2024-04-26 19:02:15.951"} PASS Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.68.177' (111) [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 41-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:17 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000b9f3f8 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 9a337250-55c5-4e07-8c95-d840cf90af2f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9a337250-55c5-4e07-8c95-d840cf90af2f {"id":"9a337250-55c5-4e07-8c95-d840cf90af2f","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2a0e 9a337250-55c5-4e07-8c95-d840cf90af2f /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348720935305222,"min-table-barrier-ts":449348720935305222,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9a337250-55c5-4e07-8c95-d840cf90af2f/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000b9f3f8 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 9a337250-55c5-4e07-8c95-d840cf90af2f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9a337250-55c5-4e07-8c95-d840cf90af2f {"id":"9a337250-55c5-4e07-8c95-d840cf90af2f","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2a0e 9a337250-55c5-4e07-8c95-d840cf90af2f /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348720935305222,"min-table-barrier-ts":449348720935305222,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9a337250-55c5-4e07-8c95-d840cf90af2f/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 + grep -q 'failed to get info:' {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000b9f3f8 TaskPositions:map[22a5b89c-4c03-47b5-9ed4-ec50f1829e9e:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} 9a337250-55c5-4e07-8c95-d840cf90af2f:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e {"id":"22a5b89c-4c03-47b5-9ed4-ec50f1829e9e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9a337250-55c5-4e07-8c95-d840cf90af2f {"id":"9a337250-55c5-4e07-8c95-d840cf90af2f","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d29e3 22a5b89c-4c03-47b5-9ed4-ec50f1829e9e /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2a0e 9a337250-55c5-4e07-8c95-d840cf90af2f /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348720935305222,"min-table-barrier-ts":449348720935305222,"admin-job-type":0} /tidb/cdc/default/default/task/position/22a5b89c-4c03-47b5-9ed4-ec50f1829e9e/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9a337250-55c5-4e07-8c95-d840cf90af2f/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "22a5b89c-4c03-47b5-9ed4-ec50f1829e9e default" | grep id "id": "9a337250-55c5-4e07-8c95-d840cf90af2f", run task successfully capture_id: 9a337250-55c5-4e07-8c95-d840cf90af2f 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '9a337250-55c5-4e07-8c95-d840cf90af2f' 9a337250-55c5-4e07-8c95-d840cf90af2f run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 161k 0 --:--:-- --:--:-- --:--:-- 171k "is_owner": false run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454ce0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40731, start at 2024-04-26 19:01:34.147824192 +0800 CST m=+4.275400007 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68454c000019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--3q21t, pid:40830, start at 2024-04-26 19:01:34.122908496 +0800 CST m=+4.183548509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:34.129 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=44910 + echo 'proxy port: 17294' proxy port: 17294 + echo 'proxy pid: 44910' proxy pid: 44910 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=17294 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f Verifying downstream PD is started... time="2024-04-26T19:02:19+08:00" level=info msg="Listening on 127.0.0.1:17294" time="2024-04-26T19:02:19+08:00" level=info msg="Not intercepting TLS connections" Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 42-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:02:19 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sinkv2/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.3508735089.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:02:13.790 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 6"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846ca880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26504, start at 2024-04-26 19:01:58.608529813 +0800 CST m=+4.366270859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.611 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.611 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {ClusterID:default ID:default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 Info:{"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} Status:0xc0021484c8 TaskPositions:map[e2b0d104-c175-4204-b7e9-72d4862fedcb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 92, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 94, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 87, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 85, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 101, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 95, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 97, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 91, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 84, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 90, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 83, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b0d104-c175-4204-b7e9-72d4862fedcb {"id":"e2b0d104-c175-4204-b7e9-72d4862fedcb","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998831 e2b0d104-c175-4204-b7e9-72d4862fedcb /tidb/cdc/default/default/changefeed/info/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} /tidb/cdc/default/default/changefeed/status/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":449348722103681027,"min-table-barrier-ts":449348722117050373,"admin-job-type":0} /tidb/cdc/default/default/task/position/e2b0d104-c175-4204-b7e9-72d4862fedcb/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {ClusterID:default ID:default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 Info:{"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} Status:0xc0021484c8 TaskPositions:map[e2b0d104-c175-4204-b7e9-72d4862fedcb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 92, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 94, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 87, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 85, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 101, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 95, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 97, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 91, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 84, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 90, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 83, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b0d104-c175-4204-b7e9-72d4862fedcb {"id":"e2b0d104-c175-4204-b7e9-72d4862fedcb","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998831 e2b0d104-c175-4204-b7e9-72d4862fedcb /tidb/cdc/default/default/changefeed/info/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} /tidb/cdc/default/default/changefeed/status/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":449348722103681027,"min-table-barrier-ts":449348722117050373,"admin-job-type":0} /tidb/cdc/default/default/task/position/e2b0d104-c175-4204-b7e9-72d4862fedcb/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {ClusterID:default ID:default/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 Info:{"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} Status:0xc0021484c8 TaskPositions:map[e2b0d104-c175-4204-b7e9-72d4862fedcb:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 92, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 94, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 87, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 85, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 101, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 95, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 97, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 91, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 84, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 90, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating tableID: 83, resolvedTs: 449348722103681027, checkpointTs: 449348722103681027, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e2b0d104-c175-4204-b7e9-72d4862fedcb {"id":"e2b0d104-c175-4204-b7e9-72d4862fedcb","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a10998831 e2b0d104-c175-4204-b7e9-72d4862fedcb /tidb/cdc/default/default/changefeed/info/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"upstream-id":7362128824110488598,"namespace":"default","changefeed-id":"c1ff9e7d-e097-4f6a-b152-06dbf46d2674","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:00.725389985+08:00","start-ts":449348700896493569,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348700922445825} /tidb/cdc/default/default/changefeed/status/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":449348722103681027,"min-table-barrier-ts":449348722117050373,"admin-job-type":0} /tidb/cdc/default/default/task/position/e2b0d104-c175-4204-b7e9-72d4862fedcb/c1ff9e7d-e097-4f6a-b152-06dbf46d2674 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128824110488598 {"id":7362128824110488598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + export http_proxy=http://127.0.0.1:17294 + http_proxy=http://127.0.0.1:17294 + export https_proxy=http://127.0.0.1:17294 + https_proxy=http://127.0.0.1:17294 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.68.177:2379' try to connect pd cluster via proxy, pd addr: 10.233.68.177:2379 + ensure 10 curl http://10.233.68.177:2379/ curl http://10.233.68.177:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 16351 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 44910 started proxy pid: 44910 + echo started proxy at port: 17294 started proxy at port: 17294 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.68.177 2379 + pd_host=10.233.68.177 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://10.233.68.177:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.45527.out cli tso query --pd=http://10.233.68.177:2379 nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 44-th check, retry later + set +x failed proxying s2c rpc error: code = Unavailable desc = client disconnected + tso='449348722632687619 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/...' + echo 449348722632687619 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449348722632687619 + echo 'query start ts: 449348722632687619' query start ts: 449348722632687619 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Fri Apr 26 19:02:23 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.4557645578.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 17294 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 17294 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Fri, 26 Apr 2024 11:02:23 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b2c80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31338, start at 2024-04-26 19:01:40.699832093 +0800 CST m=+4.450386179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.707 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.707 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6845b1e40016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--cfkdf, pid:31427, start at 2024-04-26 19:01:40.633078277 +0800 CST m=+4.326269964 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:40.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:40.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:40.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:02:24 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3550635508.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 45-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 17294 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 17294 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 616 < Content-Type: text/plain; charset=utf-8 < Date: Fri, 26 Apr 2024 11:02:26 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9bedacb-606f-4ebe-85dd-00271d1faeb1 {"id":"b9bedacb-606f-4ebe-85dd-00271d1faeb1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/39d68f1a11d5d991 b9bedacb-606f-4ebe-85dd-00271d1faeb1 /tidb/cdc/default/default/upstream/7362129224732760991 {"id":7362129224732760991,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.68.177:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9bedacb-606f-4ebe-85dd-00271d1faeb1 {"id":"b9bedacb-606f-4ebe-85dd-00271d1faeb1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/39d68f1a11d5d991 b9bedacb-606f-4ebe-85dd-00271d1faeb1 /tidb/cdc/default/default/upstream/7362129224732760991 {"id":7362129224732760991,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.68.177:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9bedacb-606f-4ebe-85dd-00271d1faeb1 {"id":"b9bedacb-606f-4ebe-85dd-00271d1faeb1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/39d68f1a11d5d991 b9bedacb-606f-4ebe-85dd-00271d1faeb1 /tidb/cdc/default/default/upstream/7362129224732760991 {"id":7362129224732760991,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.68.177:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=449348722632687619 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.45641.out cli changefeed create --start-ts=449348722632687619 --sink-uri=blackhole:/// failed proxying s2c rpc error: code = Canceled desc = context canceled Create changefeed successfully! ID: 40e18c43-74ac-41d4-9125-05545444a162 Info: {"upstream_id":7362129224732760991,"namespace":"default","id":"40e18c43-74ac-41d4-9125-05545444a162","sink_uri":"blackhole:///","create_time":"2024-04-26T19:02:27.245060514+08:00","start_ts":449348722632687619,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348722632687619,"checkpoint_ts":449348722632687619,"checkpoint_time":"2024-04-26 19:02:23.539"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 5 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:27 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:02:27 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3556735569.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 46-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:30 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:02:30 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.3564235644.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 48-th check, retry later + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq + service_type_count=6 + echo 'captured services: ' captured services: + echo . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv '[2024/04' . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /tikvpb.Tikv [2024/04 + '[' 6 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Fri Apr 26 19:02:32 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Fri Apr 26 19:02:32 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:33 GMT < Content-Length: 1141 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9cd034c9-66f8-4efc-8783-99f896198527 {"id":"9cd034c9-66f8-4efc-8783-99f896198527","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086c3 9cd034c9-66f8-4efc-8783-99f896198527 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9cd034c9-66f8-4efc-8783-99f896198527 {"id":"9cd034c9-66f8-4efc-8783-99f896198527","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086c3 9cd034c9-66f8-4efc-8783-99f896198527 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00c599a8-3084-4ac1-8fc8-b7e488780d1e {"id":"00c599a8-3084-4ac1-8fc8-b7e488780d1e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/3315c6f9-0eba-4f58-a302-521e56b29e85 {"id":"3315c6f9-0eba-4f58-a302-521e56b29e85","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/9cd034c9-66f8-4efc-8783-99f896198527 {"id":"9cd034c9-66f8-4efc-8783-99f896198527","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f0868f 00c599a8-3084-4ac1-8fc8-b7e488780d1e /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086b1 3315c6f9-0eba-4f58-a302-521e56b29e85 /tidb/cdc/default/__cdc_meta__/owner/22318f1a11f086c3 9cd034c9-66f8-4efc-8783-99f896198527 /tidb/cdc/default/default/upstream/7362129256024526555 {"id":7362129256024526555,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.cli.35707.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 0470b020-5e20-467e-bce0-bec4533c7b80 Info: {"upstream_id":7362129256024526555,"namespace":"default","id":"0470b020-5e20-467e-bce0-bec4533c7b80","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:02:33.672684605+08:00","start_ts":449348725262254092,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348725262254092,"checkpoint_ts":449348725262254092,"checkpoint_time":"2024-04-26 19:02:33.570"} PASS table ddl_manager.finish_mark not exists for 49-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 not exists for 1-th check, retry later table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null }' + echo '{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null }' { "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null } ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348723428294660, '"checkpoint_time":' '"2024-04-26' '19:02:26.574",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348723428294660, '"checkpoint_time":' '"2024-04-26' '19:02:26.574",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null }' + echo '{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null }' { "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348723428294660, "checkpoint_time": "2024-04-26 19:02:26.574", "error": null } ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348723428294660, '"checkpoint_time":' '"2024-04-26' '19:02:26.574",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348723428294660, '"checkpoint_time":' '"2024-04-26' '19:02:26.574",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348722117574662, "checkpoint_time": "2024-04-26 19:02:21.574", "error": { "time": "2024-04-26T19:02:22.702734753+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348722209062918]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348722117574662, "checkpoint_time": "2024-04-26 19:02:21.574", "error": { "time": "2024-04-26T19:02:22.702734753+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348722209062918]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7362128910180436817, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348722117574662, "checkpoint_time": "2024-04-26 19:02:21.574", "error": { "time": "2024-04-26T19:02:22.702734753+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348722209062918]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348722117574662, '"checkpoint_time":' '"2024-04-26' '19:02:21.574",' '"error":' '{' '"time":' '"2024-04-26T19:02:22.702734753+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348722209062918]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362128910180436817, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348722117574662, '"checkpoint_time":' '"2024-04-26' '19:02:21.574",' '"error":' '{' '"time":' '"2024-04-26T19:02:22.702734753+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348722209062918]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348722209062918]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:02:21.923 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348722209062918. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348722209062918]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:02:28 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later empty select id, val from test.availability2 where id=1 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6849321c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27436, start at 2024-04-26 19:02:37.988755055 +0800 CST m=+4.451935882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 52-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846cc600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26416, start at 2024-04-26 19:01:58.681603353 +0800 CST m=+4.515005167 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.680 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.680 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6846ca880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:26504, start at 2024-04-26 19:01:58.608529813 +0800 CST m=+4.366270859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:03:58.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:01:58.611 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:51:58.611 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.30369.out cli tso query --pd=http://127.0.0.1:2379 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x + tso='449348727364386817 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348727364386817 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 53-th check, retry later Starting CDC state checker... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 19:02:42 CST 2024] <<<<<< run test case multi_topics success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 54-th check, retry later + stop_proxy + kill 44910 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 19:02:43 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_kill_capture [Fri Apr 26 19:02:45 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4321143213.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_with_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 55-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } ***************** properties ***************** "updateproportion"="0" "workload"="core" "dotransactions"="false" "readallfields"="true" "mysql.user"="root" "threadcount"="10" "insertproportion"="0" "mysql.port"="4000" "operationcount"="0" "mysql.db"="consistent_replicate_storage_s3" "mysql.host"="127.0.0.1" "fieldcount"="100" "readproportion"="0" "recordcount"="5000" "requestdistribution"="uniform" "scanproportion"="0" ********************************************** [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 56-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0013793b0 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 80, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 82, resolvedTs: 449348729075138566, checkpointTs: 449348729075138562, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348729075138562,"min-table-barrier-ts":449348729075138562,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0013793b0 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 80, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 82, resolvedTs: 449348729075138566, checkpointTs: 449348729075138562, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348729075138562,"min-table-barrier-ts":449348729075138562,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0013793b0 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 80, resolvedTs: 449348729075138563, checkpointTs: 449348729075138562, state: Replicating tableID: 82, resolvedTs: 449348729075138566, checkpointTs: 449348729075138562, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348729075138562,"min-table-barrier-ts":449348729075138562,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... owner pid: 43216 owner id 69df67b1-66ea-4303-bf9a-425b5d3aafc8 table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Fri Apr 26 19:02:51 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4337743379.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 58-th check, retry later Run finished, takes 5.456563547s INSERT - Takes(s): 5.4, Count: 5000, OPS: 920.8, Avg(us): 10414, Min(us): 1787, Max(us): 104712, 95th(us): 68000, 99th(us): 91000 table ddl_manager.finish_mark not exists for 59-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:02:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0040d4348 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ccb2d070-eed5-4d91-a62e-38f9d52eeed2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ccb2d070-eed5-4d91-a62e-38f9d52eeed2 {"id":"ccb2d070-eed5-4d91-a62e-38f9d52eeed2","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ae8 ccb2d070-eed5-4d91-a62e-38f9d52eeed2 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348730647740420,"min-table-barrier-ts":449348730647740420,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ccb2d070-eed5-4d91-a62e-38f9d52eeed2/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0040d4348 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ccb2d070-eed5-4d91-a62e-38f9d52eeed2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ccb2d070-eed5-4d91-a62e-38f9d52eeed2 {"id":"ccb2d070-eed5-4d91-a62e-38f9d52eeed2","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ae8 ccb2d070-eed5-4d91-a62e-38f9d52eeed2 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348730647740420,"min-table-barrier-ts":449348730647740420,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ccb2d070-eed5-4d91-a62e-38f9d52eeed2/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc0040d4348 TaskPositions:map[69df67b1-66ea-4303-bf9a-425b5d3aafc8:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} ccb2d070-eed5-4d91-a62e-38f9d52eeed2:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/69df67b1-66ea-4303-bf9a-425b5d3aafc8 {"id":"69df67b1-66ea-4303-bf9a-425b5d3aafc8","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/ccb2d070-eed5-4d91-a62e-38f9d52eeed2 {"id":"ccb2d070-eed5-4d91-a62e-38f9d52eeed2","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ab8 69df67b1-66ea-4303-bf9a-425b5d3aafc8 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2ae8 ccb2d070-eed5-4d91-a62e-38f9d52eeed2 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348730647740420,"min-table-barrier-ts":449348730647740420,"admin-job-type":0} /tidb/cdc/default/default/task/position/69df67b1-66ea-4303-bf9a-425b5d3aafc8/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ccb2d070-eed5-4d91-a62e-38f9d52eeed2/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "69df67b1-66ea-4303-bf9a-425b5d3aafc8" | grep id table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later "id": "ccb2d070-eed5-4d91-a62e-38f9d52eeed2", "cluster-id": "default" "cluster-id": "default" run task successfully nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 61-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 62-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "operationcount"="0" "requestdistribution"="uniform" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" "mysql.user"="root" "mysql.port"="4000" "workload"="core" "readallfields"="true" "readproportion"="0" "dotransactions"="false" "scanproportion"="0" "threadcount"="4" "updateproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="50" ********************************************** Run finished, takes 21.057511ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2893.8, Avg(us): 1645, Min(us): 1013, Max(us): 4865, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table changefeed_reconstruct.usertable not exists for 2-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a91e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:33085, start at 2024-04-26 19:03:00.494821705 +0800 CST m=+4.341105017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_with_old_value/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later table partition_table.t exists table partition_table.t1 not exists for 1-th check, retry later table partition_table.t1 not exists for 2-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later check diff successfully table ddl_manager.finish_mark not exists for 64-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (39809) - No such process wait process 39809 exit for 1-th time... process 39809 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 19:03:04 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.4018740189.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting CDC state checker... table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6849321c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27436, start at 2024-04-26 19:02:37.988755055 +0800 CST m=+4.451935882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... table partition_table.t2 not exists for 2-th check, retry later wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {ClusterID:default ID:default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 Info:{"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} Status:0xc00124f2c0 TaskPositions:map[0e998f08-3dd6-455c-8934-7d4990994283:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348733883383815, checkpointTs: 449348733883383814, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e998f08-3dd6-455c-8934-7d4990994283 {"id":"0e998f08-3dd6-455c-8934-7d4990994283","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192f0d9 0e998f08-3dd6-455c-8934-7d4990994283 /tidb/cdc/default/default/changefeed/info/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} /tidb/cdc/default/default/changefeed/status/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":449348733883383814,"min-table-barrier-ts":449348733883383814,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e998f08-3dd6-455c-8934-7d4990994283/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {ClusterID:default ID:default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 Info:{"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} Status:0xc00124f2c0 TaskPositions:map[0e998f08-3dd6-455c-8934-7d4990994283:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348733883383815, checkpointTs: 449348733883383814, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e998f08-3dd6-455c-8934-7d4990994283 {"id":"0e998f08-3dd6-455c-8934-7d4990994283","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192f0d9 0e998f08-3dd6-455c-8934-7d4990994283 /tidb/cdc/default/default/changefeed/info/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} /tidb/cdc/default/default/changefeed/status/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":449348733883383814,"min-table-barrier-ts":449348733883383814,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e998f08-3dd6-455c-8934-7d4990994283/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {ClusterID:default ID:default/ac0c9269-2bb0-49ab-b57f-73388a72ec79 Info:{"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} Status:0xc00124f2c0 TaskPositions:map[0e998f08-3dd6-455c-8934-7d4990994283:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348733883383815, checkpointTs: 449348733883383814, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e998f08-3dd6-455c-8934-7d4990994283 {"id":"0e998f08-3dd6-455c-8934-7d4990994283","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1192f0d9 0e998f08-3dd6-455c-8934-7d4990994283 /tidb/cdc/default/default/changefeed/info/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"upstream-id":7362129143024775749,"namespace":"default","changefeed-id":"ac0c9269-2bb0-49ab-b57f-73388a72ec79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:02:04.525672669+08:00","start-ts":449348717617610755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348717643825155} /tidb/cdc/default/default/changefeed/status/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":449348733883383814,"min-table-barrier-ts":449348733883383814,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e998f08-3dd6-455c-8934-7d4990994283/ac0c9269-2bb0-49ab-b57f-73388a72ec79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129143024775749 {"id":7362129143024775749,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id "id": "0e998f08-3dd6-455c-8934-7d4990994283", "cluster-id": "default" run task successfully [2024/04/26 19:03:02.429 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.604064093s"] wait process cdc.test exit for 4-th time... capture_id: 0e998f08-3dd6-455c-8934-7d4990994283 check_processor_table_count http://127.0.0.1:2379 ac0c9269-2bb0-49ab-b57f-73388a72ec79 0e998f08-3dd6-455c-8934-7d4990994283 1 run task successfully check_changefeed_status 127.0.0.1:8300 47b08414-f0e1-4d36-b26a-e3686546be06 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=47b08414-f0e1-4d36-b26a-e3686546be06 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/47b08414-f0e1-4d36-b26a-e3686546be06/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 3842 0 --:--:-- --:--:-- --:--:-- 3909 + info='{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' + echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' {"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539} ++ echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 47b08414-f0e1-4d36-b26a-e3686546be06 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=47b08414-f0e1-4d36-b26a-e3686546be06 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/47b08414-f0e1-4d36-b26a-e3686546be06/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1071 0 --:--:-- --:--:-- --:--:-- 1061 100 86 100 86 0 0 1070 0 --:--:-- --:--:-- --:--:-- 1061 + info='{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' + echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' {"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539} ++ echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449348731159445539,"checkpoint_ts":449348731159445539}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later table sink_hang.t2 exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later check diff failed 5-th time, retry later check_processor_table_count http://127.0.0.1:2379 ac0c9269-2bb0-49ab-b57f-73388a72ec79 0e998f08-3dd6-455c-8934-7d4990994283 0 wait process cdc.test exit for 5-th time... table count 1 does equal to expected count 0 run task failed 1-th time, retry later wait process cdc.test exit for 6-th time... table partition_table.t2 not exists for 3-th check, retry later nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_hang_up_capture [Fri Apr 26 19:03:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4357743579.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later wait process cdc.test exit for 7-th time... cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Fri Apr 26 19:03:10 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> check diff failed 6-th time, retry later check_processor_table_count http://127.0.0.1:2379 ac0c9269-2bb0-49ab-b57f-73388a72ec79 0e998f08-3dd6-455c-8934-7d4990994283 0 run task successfully table partition_table.t2 not exists for 4-th check, retry later ***************** properties ***************** "recordcount"="50" "mysql.host"="127.0.0.1" "scanproportion"="0" "readproportion"="0" "operationcount"="0" "insertproportion"="0" "threadcount"="4" "dotransactions"="false" "workload"="core" "updateproportion"="0" "mysql.user"="root" "readallfields"="true" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.db"="changefeed_reconstruct" ********************************************** Run finished, takes 18.572998ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3581.0, Avg(us): 1486, Min(us): 911, Max(us): 5261, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later check diff failed 7-th time, retry later split_and_random_merge scale: 20 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00363eb88 TaskPositions:map[b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 84, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 80, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348734317756419,"min-table-barrier-ts":449348735117295617,"admin-job-type":0} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00363eb88 TaskPositions:map[b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 84, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 80, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348734317756419,"min-table-barrier-ts":449348735117295617,"admin-job-type":0} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00363eb88 TaskPositions:map[b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 84, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing tableID: 80, resolvedTs: 449348734317756419, checkpointTs: 449348734317756419, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backe+ grep -q 'etcd info' nd":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348734317756419,"min-table-barrier-ts":449348735117295617,"admin-job-type":0} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' "is-owner": true, run task successfully owner pid: 43582 owner id b555f7fd-6db3-48b2-a000-2ef66ad1e602 [Fri Apr 26 19:03:12 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4372143723.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later *************************** 1. row *************************** count(distinct region_id): 1 check diff failed 8-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table partition_table.finish_mark not exists for 2-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:03:15 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000c5a5b8 TaskPositions:map[490f9f5b-741a-4d26-877c-5d978d4042a5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490f9f5b-741a-4d26-877c-5d978d4042a5 {"id":"490f9f5b-741a-4d26-877c-5d978d4042a5","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b61 490f9f5b-741a-4d26-877c-5d978d4042a5 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348736152764425,"min-table-barrier-ts":449348736152764425,"admin-job-type":0} /tidb/cdc/default/default/task/position/490f9f5b-741a-4d26-877c-5d978d4042a5/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000c5a5b8 TaskPositions:map[490f9f5b-741a-4d26-877c-5d978d4042a5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490f9f5b-741a-4d26-877c-5d978d4042a5 {"id":"490f9f5b-741a-4d26-877c-5d978d4042a5","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b61 490f9f5b-741a-4d26-877c-5d978d4042a5 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348736152764425,"min-table-barrier-ts":449348736152764425,"admin-job-type":0} /tidb/cdc/default/default/task/position/490f9f5b-741a-4d26-877c-5d978d4042a5/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc000c5a5b8 TaskPositions:map[490f9f5b-741a-4d26-877c-5d978d4042a5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} b555f7fd-6db3-48b2-a000-2ef66ad1e602:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/490f9f5b-741a-4d26-877c-5d978d4042a5 {"id":"490f9f5b-741a-4d26-877c-5d978d4042a5","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/b555f7fd-6db3-48b2-a000-2ef66ad1e602 {"id":"b555f7fd-6db3-48b2-a000-2ef66ad1e602","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b41 b555f7fd-6db3-48b2-a000-2ef66ad1e602 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2b61 490f9f5b-741a-4d26-877c-5d978d4042a5 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348736152764425,"min-table-barrier-ts":449348736152764425,"admin-job-type":0} /tidb/cdc/default/default/task/position/490f9f5b-741a-4d26-877c-5d978d4042a5/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b555f7fd-6db3-48b2-a000-2ef66ad1e602/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "b555f7fd-6db3-48b2-a000-2ef66ad1e602" | grep id "cluster-id": "default" "id": "490f9f5b-741a-4d26-877c-5d978d4042a5", "cluster-id": "default" run task successfully check diff failed 9-th time, retry later table partition_table.finish_mark exists start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff successfully table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:03:17 CST 2024] <<<<<< run test case partition_table success! >>>>>> check diff failed 10-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 13-th check, retry later split_and_random_merge scale: 40 nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:03:21 CST 2024] <<<<<< run test case sink_hang success! >>>>>> table consistent_replicate_storage_s3.check1 not exists for 14-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:03:21.143 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`binlog_big`] [where="((TRUE) AND TRUE)"] ["source checksum"=214543849] ["target checksum"=0] ["get source checksum cost"=24.062268ms] ["get target checksum cost"=1.95815ms] [2024/04/26 19:03:21.673 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`binlog_big`] [where="((TRUE) AND TRUE)"] [cost=529.329791ms] [2024/04/26 19:03:21.762 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 15-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68492fd80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27360, start at 2024-04-26 19:02:37.819211012 +0800 CST m=+4.348399622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6849321c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:27436, start at 2024-04-26 19:02:37.988755055 +0800 CST m=+4.451935882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:04:37.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:02:37.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:52:37.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.31409.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348739176333313 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348739176333313 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:03:26 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.3145331455.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a91e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:33085, start at 2024-04-26 19:03:00.494821705 +0800 CST m=+4.341105017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run test case test_expire_capture [Fri Apr 26 19:03:29 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4393143933.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + workdir=/tmp/tidb_cdc_test/canal_json_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_partition_table ++ date + echo '[Fri Apr 26 19:03:27 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>>' [Fri Apr 26 19:03:27 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:30 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/365dd928-de3c-474e-aa5b-a8b54f6295d7 {"id":"365dd928-de3c-474e-aa5b-a8b54f6295d7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a12e1da8e 365dd928-de3c-474e-aa5b-a8b54f6295d7 /tidb/cdc/default/default/upstream/7362129502860346866 {"id":7362129502860346866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/365dd928-de3c-474e-aa5b-a8b54f6295d7 {"id":"365dd928-de3c-474e-aa5b-a8b54f6295d7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a12e1da8e 365dd928-de3c-474e-aa5b-a8b54f6295d7 /tidb/cdc/default/default/upstream/7362129502860346866 {"id":7362129502860346866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/365dd928-de3c-474e-aa5b-a8b54f6295d7 {"id":"365dd928-de3c-474e-aa5b-a8b54f6295d7","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a12e1da8e 365dd928-de3c-474e-aa5b-a8b54f6295d7 /tidb/cdc/default/default/upstream/7362129502860346866 {"id":7362129502860346866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.31514.out cli changefeed create --start-ts=449348739176333313 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 89204d92-b571-4e91-93b3-b38afb0a2009 Info: {"upstream_id":7362129502860346866,"namespace":"default","id":"89204d92-b571-4e91-93b3-b38afb0a2009","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:03:30.361486844+08:00","start_ts":449348739176333313,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348739176333313,"checkpoint_ts":449348739176333313,"checkpoint_time":"2024-04-26 19:03:26.648"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002019da0 TaskPositions:map[68ea6eab-b16c-4b74-966e-9df7f776ee33:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating tableID: 82, resolvedTs: 449348740622319621, checkpointTs: 449348740622319618, state: Replicating tableID: 84, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68ea6eab-b16c-4b74-966e-9df7f776ee33 {"id":"68ea6eab-b16c-4b74-966e-9df7f776ee33","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bb8 68ea6eab-b16c-4b74-966e-9df7f776ee33 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348740622319618,"min-table-barrier-ts":449348740622319618,"admin-job-type":0} /tidb/cdc/default/default/task/position/68ea6eab-b16c-4b74-966e-9df7f776ee33/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002019da0 TaskPositions:map[68ea6eab-b16c-4b74-966e-9df7f776ee33:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating tableID: 82, resolvedTs: 449348740622319621, checkpointTs: 449348740622319618, state: Replicating tableID: 84, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68ea6eab-b16c-4b74-966e-9df7f776ee33 {"id":"68ea6eab-b16c-4b74-966e-9df7f776ee33","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bb8 68ea6eab-b16c-4b74-966e-9df7f776ee33 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348740622319618,"min-table-barrier-ts":449348740622319618,"admin-job-type":0} /tidb/cdc/default/default/task/position/68ea6eab-b16c-4b74-966e-9df7f776ee33/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc002019da0 TaskPositions:map[68ea6eab-b16c-4b74-966e-9df7f776ee33:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 80, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating tableID: 82, resolvedTs: 449348740622319621, checkpointTs: 449348740622319618, state: Replicating tableID: 84, resolvedTs: 449348740622319619, checkpointTs: 449348740622319618, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68ea6eab-b16c-4b74-966e-9df7f776ee33 {"id":"68ea6eab-b16c-4b74-966e-9df7f776ee33","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bb8 68ea6eab-b16c-4b74-966e-9df7f776ee33 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file+ grep -q 'etcd info' -backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348740622319618,"min-table-barrier-ts":449348740622319618,"admin-job-type":0} /tidb/cdc/default/default/task/position/68ea6eab-b16c-4b74-966e-9df7f776ee33/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... "is-owner": true, run task successfully table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later owner pid: 43936 owner id 68ea6eab-b16c-4b74-966e-9df7f776ee33 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '68ea6eab-b16c-4b74-966e-9df7f776ee33' /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bb8 run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7d34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25256, start at 2024-04-26 19:03:31.927746211 +0800 CST m=+4.387346631 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later table partition_table.t not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Fri Apr 26 19:03:36 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.4414944151.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_s3.t2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully ***************** properties ***************** "workload"="core" "mysql.host"="127.0.0.1" "threadcount"="2" "readallfields"="true" "recordcount"="10" "dotransactions"="false" "requestdistribution"="uniform" "mysql.user"="root" "mysql.port"="4000" "insertproportion"="0" "readproportion"="0" "scanproportion"="0" "mysql.db"="multi_capture_1" "operationcount"="0" "updateproportion"="0" ********************************************** Run finished, takes 10.291883ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1791.5, Avg(us): 1965, Min(us): 1090, Max(us): 4576, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "mysql.db"="multi_capture_2" "threadcount"="2" "operationcount"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "recordcount"="10" "scanproportion"="0" ********************************************** Run finished, takes 10.376824ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1836.5, Avg(us): 1988, Min(us): 1014, Max(us): 4866, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readproportion"="0" "updateproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "mysql.user"="root" "readallfields"="true" "insertproportion"="0" "recordcount"="10" "operationcount"="0" "mysql.port"="4000" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="multi_capture_3" "workload"="core" ********************************************** Run finished, takes 10.636874ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1693.9, Avg(us): 2045, Min(us): 1335, Max(us): 4651, 95th(us): 5000, 99th(us): 5000 table partition_table.t not exists for 2-th check, retry later ***************** properties ***************** "updateproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "mysql.port"="4000" "recordcount"="10" "mysql.user"="root" "threadcount"="2" "dotransactions"="false" "readproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "insertproportion"="0" "scanproportion"="0" "mysql.db"="multi_capture_4" "operationcount"="0" ********************************************** Run finished, takes 9.828269ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1812.9, Avg(us): 1842, Min(us): 1146, Max(us): 4204, 95th(us): 5000, 99th(us): 5000 [Fri Apr 26 19:03:38 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3056430566.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:03:38 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.3768837690.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd17c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41541, start at 2024-04-26 19:03:37.34746794 +0800 CST m=+4.460950026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_with_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00328b308 TaskPositions:map[9089bd9e-784d-446e-879e-0682cbe0f0d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 80, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 82, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089bd9e-784d-446e-879e-0682cbe0f0d1 {"id":"9089bd9e-784d-446e-879e-0682cbe0f0d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bf0 9089bd9e-784d-446e-879e-0682cbe0f0d1 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348741409013762,"min-table-barrier-ts":449348742457327618,"admin-job-type":0} /tidb/cdc/default/default/task/position/9089bd9e-784d-446e-879e-0682cbe0f0d1/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00328b308 TaskPositions:map[9089bd9e-784d-446e-879e-0682cbe0f0d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 80, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 82, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089bd9e-784d-446e-879e-0682cbe0f0d1 {"id":"9089bd9e-784d-446e-879e-0682cbe0f0d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bf0 9089bd9e-784d-446e-879e-0682cbe0f0d1 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backe+ grep -q 'failed to get info:' nd":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348741409013762,"min-table-barrier-ts":449348742457327618,"admin-job-type":0} /tidb/cdc/default/default/task/position/9089bd9e-784d-446e-879e-0682cbe0f0d1/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3990a444-9a6b-45c7-9bc3-bbb98798b335 {ClusterID:default ID:default/3990a444-9a6b-45c7-9bc3-bbb98798b335 Info:{"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} Status:0xc00328b308 TaskPositions:map[9089bd9e-784d-446e-879e-0682cbe0f0d1:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 80, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing tableID: 82, resolvedTs: 449348741409013762, checkpointTs: 449348741409013762, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089bd9e-784d-446e-879e-0682cbe0f0d1 {"id":"9089bd9e-784d-446e-879e-0682cbe0f0d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a0f6d2bf0 9089bd9e-784d-446e-879e-0682cbe0f0d1 /tidb/cdc/default/default/changefeed/info/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"upstream-id":7362128520909889718,"namespace":"default","changefeed-id":"3990a444-9a6b-45c7-9bc3-bbb98798b335","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:00:41.451914742+08:00","start-ts":449348679791804417,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348695861493768} /tidb/cdc/default/default/changefeed/status/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":449348741409013762,"min-table-barrier-ts":449348742457327618,"admin-job-type":0} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/9089bd9e-784d-446e-879e-0682cbe0f0d1/3990a444-9a6b-45c7-9bc3-bbb98798b335 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362128520909889718 {"id":7362128520909889718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later owner pid: 44154 owner id 9089bd9e-784d-446e-879e-0682cbe0f0d1 changefeed id: 3990a444-9a6b-45c7-9bc3-bbb98798b335 curl -s -d "cf-id=3990a444-9a6b-45c7-9bc3-bbb98798b335&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 628 969 --:--:-- --:--:-- --:--:-- 981 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf544001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:35041, start at 2024-04-26 19:03:39.646085664 +0800 CST m=+4.252476192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:41 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:03:41 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3062230624.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:41 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {ClusterID:default ID:default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 Info:{"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} Status:0xc002eb51b8 TaskPositions:map[d8d28430-118b-46a7-9322-b1115ea49fab:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 84, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 82, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 89, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 91, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 92, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 87, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 88, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8d28430-118b-46a7-9322-b1115ea49fab {"id":"d8d28430-118b-46a7-9322-b1115ea49fab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e9478 d8d28430-118b-46a7-9322-b1115ea49fab /tidb/cdc/default/default/changefeed/info/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} /tidb/cdc/default/default/changefeed/status/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":449348742704005122,"min-table-barrier-ts":449348742717112324,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8d28430-118b-46a7-9322-b1115ea49fab/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {ClusterID:default ID:default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 Info:{"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} Status:0xc002eb51b8 TaskPositions:map[d8d28430-118b-46a7-9322-b1115ea49fab:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 84, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 82, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 89, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 91, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 92, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 87, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 88, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8d28430-118b-46a7-9322-b1115ea49fab {"id":"d8d28430-118b-46a7-9322-b1115ea49fab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e9478 d8d28430-118b-46a7-9322-b1115ea49fab /tidb/cdc/default/default/changefeed/info/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} /tidb/cdc/default/default/changefeed/status/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":449348742704005122,"min-table-barrier-ts":449348742717112324,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8d28430-118b-46a7-9322-b1115ea49fab/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {ClusterID:default ID:default/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 Info:{"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} Status:0xc002eb51b8 TaskPositions:map[d8d28430-118b-46a7-9322-b1115ea49fab:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 94, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 84, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 82, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 89, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 91, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 92, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 87, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating tableID: 88, resolvedTs: 449348742704005122, checkpointTs: 449348742704005122, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8d28430-118b-46a7-9322-b1115ea49fab {"id":"d8d28430-118b-46a7-9322-b1115ea49fab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a115e9478 d8d28430-118b-46a7-9322-b1115ea49fab /tidb/cdc/default/default/changefeed/info/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"upstream-id":7362129093579252335,"namespace":"default","changefeed-id":"9a104ed1-0010-4c3b-95fb-65d6d70c3b40","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:01:51.053148269+08:00","start-ts":449348714090987524,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348714104094727} /tidb/cdc/default/default/changefeed/status/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":449348742704005122,"min-table-barrier-ts":449348742717112324,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8d28430-118b-46a7-9322-b1115ea49fab/9a104ed1-0010-4c3b-95fb-65d6d70c3b40 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129093579252335 {"id":7362129093579252335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting CDC state checker... Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log table partition_table.finish_mark not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2533 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:03:29.179","puller_resolved_ts":"2024-04-26 19:03:29.179","last_synced_ts":"2024-04-26 19:01:13.029","now_ts":"2024-04-26 19:03:30.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:03:29.179","puller_resolved_ts":"2024-04-26' '19:03:29.179","last_synced_ts":"2024-04-26' '19:01:13.029","now_ts":"2024-04-26' '19:03:30.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 25169 6.6 0.0 12989384 154748 ? Sl 19:00 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 25224 3.7 0.0 12930828 141572 ? Sl 19:00 0:07 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-04-26T19:03:37.881+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:37.885+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:37.980+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:03:43 CST 2024] <<<<<< run test case availability success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... table partition_table.finish_mark not exists for 3-th check, retry later {"level":"warn","ts":"2024-04-26T19:03:43.883+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:43.886+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:43.981+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} [2024/04/26 19:03:42.656 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=4111818282] ["target checksum"=2174054685] ["get source checksum cost"=3.141456ms] ["get target checksum cost"=2.429735ms] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:44 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:03:44 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.3067430676.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table partition_table.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d39500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37489, start at 2024-04-26 19:03:43.961499353 +0800 CST m=+4.566079665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:47 GMT < Content-Length: 1141 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 {"id":"68da70fb-c11e-4e1b-8d61-a6562c5bb1b7","address":"127.0.0.1:8303","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8080 68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 {"id":"68da70fb-c11e-4e1b-8d61-a6562c5bb1b7","address":"127.0.0.1:8303","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8080 68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1ccd8cf3-cd4b-449e-bed1-fea300e2f31e {"id":"1ccd8cf3-cd4b-449e-bed1-fea300e2f31e","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/4abf9334-a654-4e1b-93ef-64664a80f8aa {"id":"4abf9334-a654-4e1b-93ef-64664a80f8aa","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 {"id":"68da70fb-c11e-4e1b-8d61-a6562c5bb1b7","address":"127.0.0.1:8303","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f804b 4abf9334-a654-4e1b-93ef-64664a80f8aa /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8075 1ccd8cf3-cd4b-449e-bed1-fea300e2f31e /tidb/cdc/default/__cdc_meta__/owner/22318f1a122f8080 68da70fb-c11e-4e1b-8d61-a6562c5bb1b7 /tidb/cdc/default/default/upstream/7362129324134944524 {"id":7362129324134944524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.30729.out cli changefeed create --start-ts=449348727364386817 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 table partition_table.finish_mark not exists for 5-th check, retry later Starting CDC state checker... Create changefeed successfully! ID: 9f929410-967b-4c12-8377-2bc7cbce7755 Info: {"upstream_id":7362129324134944524,"namespace":"default","id":"9f929410-967b-4c12-8377-2bc7cbce7755","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:03:47.694789975+08:00","start_ts":449348727364386817,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348727364386817,"checkpoint_ts":449348727364386817,"checkpoint_time":"2024-04-26 19:02:41.589"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table multi_capture_1.usertable not exists for 1-th check, retry later {"level":"warn","ts":"2024-04-26T19:03:47.871+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:03:47.872+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:03:47.872+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:03:47.872+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:03:47.964+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:03:47.964+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714129427.9840415,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0015dce00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714129427.9841065,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714129427.9842446,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017e48c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714129427.984292,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714129428.0715852,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ace000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714129428.0716472,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a8fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:32993, start at 2024-04-26 19:03:00.352540664 +0800 CST m=+4.253749570 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684a91e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:33085, start at 2024-04-26 19:03:00.494821705 +0800 CST m=+4.341105017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:00.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:00.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:00.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:03:48 CST 2024] <<<<<< START cdc server in new_ci_collation_with_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_with_old_value.3710637108.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_with_old_value/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later {"level":"warn","ts":"2024-04-26T19:03:49.884+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:49.887+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:49.982+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} table partition_table.finish_mark not exists for 6-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:03:51 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6988f79d-2d35-427a-bc05-a0ad5f8e9df5 {"id":"6988f79d-2d35-427a-bc05-a0ad5f8e9df5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a133b208e 6988f79d-2d35-427a-bc05-a0ad5f8e9df5 /tidb/cdc/default/default/upstream/7362129594991697678 {"id":7362129594991697678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6988f79d-2d35-427a-bc05-a0ad5f8e9df5 {"id":"6988f79d-2d35-427a-bc05-a0ad5f8e9df5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a133b208e 6988f79d-2d35-427a-bc05-a0ad5f8e9df5 /tidb/cdc/default/default/upstream/7362129594991697678 {"id":7362129594991697678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6988f79d-2d35-427a-bc05-a0ad5f8e9df5 {"id":"6988f79d-2d35-427a-bc05-a0ad5f8e9df5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a133b208e 6988f79d-2d35-427a-bc05-a0ad5f8e9df5 /tidb/cdc/default/default/upstream/7362129594991697678 {"id":7362129594991697678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 68c86bd3-2d57-4aa7-8c01-2807be9bd48e Info: {"upstream_id":7362129594991697678,"namespace":"default","id":"68c86bd3-2d57-4aa7-8c01-2807be9bd48e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-04-26T19:03:52.116887185+08:00","start_ts":449348744997240833,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348744997240833,"checkpoint_ts":449348744997240833,"checkpoint_time":"2024-04-26 19:03:48.853"} start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 8-th check, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table.finish_mark not exists for 9-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:03:55.885+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:55.888+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:03:55.983+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7d34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25256, start at 2024-04-26 19:03:31.927746211 +0800 CST m=+4.387346631 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table multi_capture_4.usertable exists check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff successfully ***************** properties ***************** "recordcount"="20" "insertproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "dotransactions"="false" "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.db"="multi_capture_1" "operationcount"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "workload"="core" ********************************************** Run finished, takes 12.673216ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1759.9, Avg(us): 1211, Min(us): 573, Max(us): 1974, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "readproportion"="0" "mysql.user"="root" "scanproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "readallfields"="true" "mysql.db"="multi_capture_2" "recordcount"="20" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "dotransactions"="false" "insertproportion"="0" "operationcount"="0" "mysql.port"="4000" ********************************************** Run finished, takes 13.310567ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1758.3, Avg(us): 1250, Min(us): 658, Max(us): 2070, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "updateproportion"="0" "workload"="core" "insertproportion"="0" "operationcount"="0" "readallfields"="true" "mysql.port"="4000" "scanproportion"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "recordcount"="20" "dotransactions"="false" "mysql.user"="root" "mysql.db"="multi_capture_3" "threadcount"="2" "requestdistribution"="uniform" ********************************************** Run finished, takes 13.310266ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1658.1, Avg(us): 1151, Min(us): 591, Max(us): 1844, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "dotransactions"="false" "mysql.user"="root" "recordcount"="20" "readproportion"="0" "threadcount"="2" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "updateproportion"="0" "operationcount"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_capture_4" "insertproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 12.934936ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1787.9, Avg(us): 1210, Min(us): 560, Max(us): 1950, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd17c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41541, start at 2024-04-26 19:03:37.34746794 +0800 CST m=+4.460950026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.finish_mark not exists for 11-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:04:01 CST 2024] <<<<<< run test case multi_capture success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:04:01.886+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:01.889+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:01.985+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} table partition_table.finish_mark not exists for 12-th check, retry later check_redo_resolved_ts 9a104ed1-0010-4c3b-95fb-65d6d70c3b40 449348748838174726 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/9a104ed1-0010-4c3b-95fb-65d6d70c3b40/meta start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf544001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:35041, start at 2024-04-26 19:03:39.646085664 +0800 CST m=+4.252476192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) global resolved ts 449348747987255299 not forward to 449348748838174726 run task failed 1-th time, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 13-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... check_redo_resolved_ts 9a104ed1-0010-4c3b-95fb-65d6d70c3b40 449348748838174726 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/9a104ed1-0010-4c3b-95fb-65d6d70c3b40/meta run task successfully table partition_table.finish_mark not exists for 14-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:04:06.339 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:04:06.340 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:04:06.343 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:04:06.343 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348743202078731] [resolvedTs=449348749008568325] [2024/04/26 19:04:06.344 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:04:06.344 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:04:06.344 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:04:06.345 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:04:06.349 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:04:06.351 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=7.260561ms] [2024/04/26 19:04:06.352 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.355 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:04:06.356 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:04:06.357 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:04:06.357 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:04:06.357 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:04:06.362 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:04:06.368 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d39500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37489, start at 2024-04-26 19:03:43.961499353 +0800 CST m=+4.566079665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 19:04:07.227 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=883.65747ms] [2024/04/26 19:04:07.228 +08:00] [ERROR] [redo.go:265] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449348743202078726,\"CommitTs\":449348743202078731,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_s3\",\"tbl-name\":\"t1\",\"tbl-id\":86,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table partition_table.finish_mark not exists for 15-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:04:07.887+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:07.890+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:07.985+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:04:08.792 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348749008568325] [2024/04/26 19:04:08.792 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348749008568325] [2024/04/26 19:04:08.893 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348749008568325] [2024/04/26 19:04:08.893 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348749008568325] [2024/04/26 19:04:09.596 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=449348749008568325] [2024/04/26 19:04:09.596 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=96] [checkpointTs=449348749008568325] table partition_table.finish_mark exists [2024/04/26 19:04:09.696 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449348749008568325] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 19:04:09.696 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 19:04:09.697 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully check diff successfully check diff successfully [Fri Apr 26 19:04:10 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:04:10 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:04:13.888+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:13.891+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:13.987+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef9980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45320, start at 2024-04-26 19:04:12.674825513 +0800 CST m=+4.239969501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7b9c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25194, start at 2024-04-26 19:03:31.847617181 +0800 CST m=+4.384980102 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684c7d34000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:25256, start at 2024-04-26 19:03:31.927746211 +0800 CST m=+4.387346631 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:31.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:31.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:31.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:04:14 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.2909129093.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:04:17 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68f3efc-55d2-417f-9ed1-4f9adaf6211e {"id":"c68f3efc-55d2-417f-9ed1-4f9adaf6211e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04b88 c68f3efc-55d2-417f-9ed1-4f9adaf6211e /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68f3efc-55d2-417f-9ed1-4f9adaf6211e {"id":"c68f3efc-55d2-417f-9ed1-4f9adaf6211e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04b88 c68f3efc-55d2-417f-9ed1-4f9adaf6211e /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c68f3efc-55d2-417f-9ed1-4f9adaf6211e {"id":"c68f3efc-55d2-417f-9ed1-4f9adaf6211e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04b88 c68f3efc-55d2-417f-9ed1-4f9adaf6211e /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.29160.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7362129734779594518,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-04-26T19:04:18.197043504+08:00","start_ts":449348752651059206,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348752651059206,"checkpoint_ts":449348752651059206,"checkpoint_time":"2024-04-26 19:04:18.050"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd23c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41443, start at 2024-04-26 19:03:37.372883811 +0800 CST m=+4.567313750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.381 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cd17c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:41541, start at 2024-04-26 19:03:37.34746794 +0800 CST m=+4.460950026 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:37.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:37.311 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:37.311 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:04:19 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.4563945641.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:04:19.889+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012ec1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:19.892+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f2c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:04:19.989+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007c2000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:04:22 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb53f8d2-ed17-4e84-bca0-e50e18117fdd {"id":"cb53f8d2-ed17-4e84-bca0-e50e18117fdd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21d88 cb53f8d2-ed17-4e84-bca0-e50e18117fdd /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb53f8d2-ed17-4e84-bca0-e50e18117fdd {"id":"cb53f8d2-ed17-4e84-bca0-e50e18117fdd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21d88 cb53f8d2-ed17-4e84-bca0-e50e18117fdd /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb53f8d2-ed17-4e84-bca0-e50e18117fdd {"id":"cb53f8d2-ed17-4e84-bca0-e50e18117fdd","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21d88 cb53f8d2-ed17-4e84-bca0-e50e18117fdd /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf888000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:34940, start at 2024-04-26 19:03:39.819612013 +0800 CST m=+4.495880807 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.825 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.810 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.810 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684cf544001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:35041, start at 2024-04-26 19:03:39.646085664 +0800 CST m=+4.252476192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:39.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:39.651 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:39.651 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31919, start at 2024-04-26 19:04:23.887454395 +0800 CST m=+4.722005959 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.858 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.858 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log table drop_tables.c not exists for 1-th check, retry later table drop_tables.c not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d37b8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37401, start at 2024-04-26 19:03:43.874066455 +0800 CST m=+4.527984434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684d39500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:37489, start at 2024-04-26 19:03:43.961499353 +0800 CST m=+4.566079665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:05:43.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:03:43.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:53:43.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41593.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348755163971585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348755163971585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/04/26 19:04:23.996 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:04:23.997 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:04:23.997 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348722103681027] [resolvedTs=449348753587437573] [2024/04/26 19:04:23.997 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:04:23.997 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:04:23.998 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=998.72Β΅s] [2024/04/26 19:04:23.998 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=1.062209ms] [2024/04/26 19:04:24.003 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:04:24.005 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.006 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:04:24.007 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:04:24.007 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:04:24.007 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:04:24.011 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:04:24.012 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 19:04:24.013 +08:00] [WARN] [redo.go:280] ["apply DDL"] [ddl="{\"StartTs\":449348722470944776,\"CommitTs\":449348722470944780,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/04/26 19:04:27.424 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":449348722470944776,\"CommitTs\":449348722470944780,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/04/26 19:04:27.627 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="alter table t1 drop partition p1"] [duration=203.395678ms] [namespace=] [changefeed=] [2024/04/26 19:04:27.627 +08:00] [WARN] [redo.go:280] ["apply DDL"] [ddl="{\"StartTs\":449348722510266376,\"CommitTs\":449348722510266381,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/04/26 19:04:27.827 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":449348722510266376,\"CommitTs\":449348722510266381,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/04/26 19:04:27.979 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [duration=151.36464ms] [namespace=] [changefeed=] [2024/04/26 19:04:27.979 +08:00] [WARN] [redo.go:280] ["apply DDL"] [ddl="{\"StartTs\":449348722549325832,\"CommitTs\":449348722549325837,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/04/26 19:04:28.380 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":449348722549325832,\"CommitTs\":449348722549325837,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"partition_table\",\"tbl-name\":\"t1\",\"tbl-id\":89,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/04/26 19:04:28.522 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [duration=141.719247ms] [namespace=] [changefeed=] [2024/04/26 19:04:28.522 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=449348753587437573] [2024/04/26 19:04:28.522 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=83] [checkpointTs=449348753587437573] [2024/04/26 19:04:28.623 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=449348753587437573] [2024/04/26 19:04:28.623 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=92] [checkpointTs=449348753587437573] [2024/04/26 19:04:28.724 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348753587437573] [2024/04/26 19:04:28.724 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=94] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.026 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.026 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=97] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.327 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.327 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=101] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.428 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.428 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=85] [checkpointTs=449348753587437573] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:04:29.530 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.530 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=87] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.630 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.630 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=84] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.731 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.731 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=449348753587437573] table drop_tables.c not exists for 3-th check, retry later [2024/04/26 19:04:29.832 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.832 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=90] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.932 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348753587437573] [2024/04/26 19:04:29.932 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=91] [checkpointTs=449348753587437573] [2024/04/26 19:04:30.033 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=449348753587437573] [2024/04/26 19:04:30.033 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=95] [checkpointTs=449348753587437573] [2024/04/26 19:04:30.134 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=45] [appliedDDLCount=3] [currentCheckpoint=449348753587437573] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:04:30.134 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully check diff successfully [Fri Apr 26 19:04:30 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 test chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table drop_tables.c not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c exists check diff successfully Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:04:36 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685075040019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:39071, start at 2024-04-26 19:04:36.968820951 +0800 CST m=+9.506507543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.979 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.979 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:04:31.855 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=49.199129513s] [2024/04/26 19:04:31.858 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2024/04/26 19:04:37.646 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=4111818282] ["get source checksum cost"=1.3798ms] ["get target checksum cost"=4.399041ms] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef9980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45320, start at 2024-04-26 19:04:12.674825513 +0800 CST m=+4.239969501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a7e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27538, start at 2024-04-26 19:04:40.192076476 +0800 CST m=+4.337359163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... Verifying downstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31919, start at 2024-04-26 19:04:23.887454395 +0800 CST m=+4.722005959 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.858 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.858 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685170cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36321, start at 2024-04-26 19:04:53.067934959 +0800 CST m=+5.353551115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:53.073 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:53.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:53.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log table new_ci_collation_with_old_value_test.t1 exists table new_ci_collation_with_old_value_test.t2 exists table new_ci_collation_with_old_value_test.t3 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t3 exists table new_ci_collation_with_old_value_test.t4 not exists for 1-th check, retry later table new_ci_collation_with_old_value_test.t4 not exists for 2-th check, retry later table new_ci_collation_with_old_value_test.t4 exists table new_ci_collation_with_old_value_test.t5 exists check diff failed 1-th time, retry later check diff successfully check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:05:01 CST 2024] <<<<<< run test case new_ci_collation_with_old_value success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851eb000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32632, start at 2024-04-26 19:05:00.880259273 +0800 CST m=+4.257608988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685075040019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:39071, start at 2024-04-26 19:04:36.968820951 +0800 CST m=+9.506507543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.979 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.979 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef8e0001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45234, start at 2024-04-26 19:04:12.649181822 +0800 CST m=+4.273625300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684ef9980016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--q3xg2, pid:45320, start at 2024-04-26 19:04:12.674825513 +0800 CST m=+4.239969501 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:12.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:12.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:12.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a7e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27538, start at 2024-04-26 19:04:40.192076476 +0800 CST m=+4.337359163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8d8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31844, start at 2024-04-26 19:04:23.883154257 +0800 CST m=+4.780278130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c684fa8c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:31919, start at 2024-04-26 19:04:23.887454395 +0800 CST m=+4.722005959 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:23.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:23.858 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:23.858 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.35712.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348765979508737 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348765979508737 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/new_ci_collation_without_old_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table1 not exists for 1-th check, retry later Verifying downstream PD is started... table test.table1 exists table test.table2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.table2 exists check diff failed 1-th time, retry later Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685072c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:38978, start at 2024-04-26 19:04:36.817607012 +0800 CST m=+9.442887869 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.786 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.786 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685075040019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:39071, start at 2024-04-26 19:04:36.968820951 +0800 CST m=+9.506507543 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:36.975 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:36.979 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:36.979 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:05:22 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.4320543207.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a66c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27453, start at 2024-04-26 19:04:40.130193714 +0800 CST m=+4.362721568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6850a7e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:27538, start at 2024-04-26 19:04:40.192076476 +0800 CST m=+4.337359163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:40.198 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.31520.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... check diff successfully Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" "mysql.port"="4000" "workload"="core" "readproportion"="0" "recordcount"="10" "insertproportion"="0" "mysql.db"="sink_retry" "threadcount"="2" "updateproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 11.51118ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1669.7, Avg(us): 2179, Min(us): 1159, Max(us): 5644, 95th(us): 6000, 99th(us): 6000 [Fri Apr 26 19:05:24 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.4172641728.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449348769910095873 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348769910095873 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449348769910095873 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:05:24 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3156231564.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Resume changefeed with checkpointTs3 449348770035662851 table test.table5 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:25 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9078df20-1259-4c20-8264-7f9ce2d7605a {"id":"9078df20-1259-4c20-8264-7f9ce2d7605a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a72c8e 9078df20-1259-4c20-8264-7f9ce2d7605a /tidb/cdc/default/default/upstream/7362129991609131683 {"id":7362129991609131683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9078df20-1259-4c20-8264-7f9ce2d7605a {"id":"9078df20-1259-4c20-8264-7f9ce2d7605a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a72c8e 9078df20-1259-4c20-8264-7f9ce2d7605a /tidb/cdc/default/default/upstream/7362129991609131683 {"id":7362129991609131683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9078df20-1259-4c20-8264-7f9ce2d7605a {"id":"9078df20-1259-4c20-8264-7f9ce2d7605a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a72c8e 9078df20-1259-4c20-8264-7f9ce2d7605a /tidb/cdc/default/default/upstream/7362129991609131683 {"id":7362129991609131683,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.43272.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851eb000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32632, start at 2024-04-26 19:05:00.880259273 +0800 CST m=+4.257608988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Create changefeed successfully! ID: a150e42c-0f4e-45fb-81c3-116a21c543c1 Info: {"upstream_id":7362129991609131683,"namespace":"default","id":"a150e42c-0f4e-45fb-81c3-116a21c543c1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-04-26T19:05:25.956753987+08:00","start_ts":449348770423898115,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348770423898115,"checkpoint_ts":449348770423898115,"checkpoint_time":"2024-04-26 19:05:25.848"} PASS ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later table ddl_only_block_related_table.finish_mark exists /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (29097) - No such process wait process 29097 exit for 1-th time... process 29097 already exit [Fri Apr 26 19:05:22 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.2929529297.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc002952948 TaskPositions:map[ffe7c12d-051d-42ad-bbe5-750310e74ce6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 88, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 82, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 84, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe7c12d-051d-42ad-bbe5-750310e74ce6 {"id":"ffe7c12d-051d-42ad-bbe5-750310e74ce6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04cc5 ffe7c12d-051d-42ad-bbe5-750310e74ce6 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770149171205,"min-table-barrier-ts":449348770149171205,"admin-job-type":0} /tidb/cdc/default/default/task/position/ffe7c12d-051d-42ad-bbe5-750310e74ce6/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc002952948 TaskPositions:map[ffe7c12d-051d-42ad-bbe5-750310e74ce6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 88, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 82, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 84, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe7c12d-051d-42ad-bbe5-750310e74ce6 {"id":"ffe7c12d-051d-42ad-bbe5-750310e74ce6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04cc5 ffe7c12d-051d-42ad-bbe5-750310e74ce6 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770149171205,"min-table-barrier-ts":449348770149171205,"admin-job-type":0} /tidb/cdc/default/default/task/position/ffe7c12d-051d-42ad-bbe5-750310e74ce6/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc002952948 TaskPositions:map[ffe7c12d-051d-42ad-bbe5-750310e74ce6:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 86, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 88, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 82, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating tableID: 84, resolvedTs: 449348770149171205, checkpointTs: 449348770149171205, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe7c12d-051d-42ad-bbe5-750310e74ce6 {"id":"ffe7c12d-051d-42ad-bbe5-750310e74ce6","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04cc5 ffe7c12d-051d-42ad-bbe5-750310e74ce6 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770149171205,"min-table-barrier-ts":449348770149171205,"admin-job-type":0} /tidb/cdc/default/default/task/position/ffe7c12d-051d-42ad-bbe5-750310e74ce6/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x check diff failed 1-th time, retry later table test.table5 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:27 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/428515cc-9fcb-4f75-a057-24b44fc320b2 {"id":"428515cc-9fcb-4f75-a057-24b44fc320b2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13cdbe0e 428515cc-9fcb-4f75-a057-24b44fc320b2 /tidb/cdc/default/default/upstream/7362129784427713754 {"id":7362129784427713754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/428515cc-9fcb-4f75-a057-24b44fc320b2 {"id":"428515cc-9fcb-4f75-a057-24b44fc320b2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13cdbe0e 428515cc-9fcb-4f75-a057-24b44fc320b2 /tidb/cdc/default/default/upstream/7362129784427713754 {"id":7362129784427713754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/428515cc-9fcb-4f75-a057-24b44fc320b2 {"id":"428515cc-9fcb-4f75-a057-24b44fc320b2","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13cdbe0e 428515cc-9fcb-4f75-a057-24b44fc320b2 /tidb/cdc/default/default/upstream/7362129784427713754 {"id":7362129784427713754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.41784.out cli changefeed create --start-ts=449348755163971585 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: a41af0c5-8e14-42fe-b752-f35bf50ababa Info: {"upstream_id":7362129784427713754,"namespace":"default","id":"a41af0c5-8e14-42fe-b752-f35bf50ababa","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:05:27.476745328+08:00","start_ts":449348755163971585,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348755163971585,"checkpoint_ts":449348755163971585,"checkpoint_time":"2024-04-26 19:04:27.636"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:27 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b36a98d-9e46-4836-a1c2-dd7253ad12af {"id":"5b36a98d-9e46-4836-a1c2-dd7253ad12af","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a9e18e 5b36a98d-9e46-4836-a1c2-dd7253ad12af /tidb/cdc/default/default/upstream/7362130026612654698 {"id":7362130026612654698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b36a98d-9e46-4836-a1c2-dd7253ad12af {"id":"5b36a98d-9e46-4836-a1c2-dd7253ad12af","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a9e18e 5b36a98d-9e46-4836-a1c2-dd7253ad12af /tidb/cdc/default/default/upstream/7362130026612654698 {"id":7362130026612654698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b36a98d-9e46-4836-a1c2-dd7253ad12af {"id":"5b36a98d-9e46-4836-a1c2-dd7253ad12af","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a14a9e18e 5b36a98d-9e46-4836-a1c2-dd7253ad12af /tidb/cdc/default/default/upstream/7362130026612654698 {"id":7362130026612654698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449348769910095873 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.31628.out cli changefeed create --start-ts=449348769910095873 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362130026612654698,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:05:27.581788289+08:00","start_ts":449348769910095873,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348769910095873,"checkpoint_ts":449348769910095873,"checkpoint_time":"2024-04-26 19:05:23.888"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' table test.table5 exists table test.table6 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685381f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38450, start at 2024-04-26 19:05:26.924531603 +0800 CST m=+4.276848453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:26.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:26.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:26.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation_without_old_value/tiflash/log/error.log check diff successfully check_ts_not_forward ddl-only-block-related-table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table6 exists check diff failed 1-th time, retry later [Fri Apr 26 19:05:23 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.3942039422.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:26 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b29745ed-9098-46fe-910c-5b80b201615e {"id":"b29745ed-9098-46fe-910c-5b80b201615e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13bf337a b29745ed-9098-46fe-910c-5b80b201615e /tidb/cdc/default/default/upstream/7362129765043607146 {"id":7362129765043607146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b29745ed-9098-46fe-910c-5b80b201615e {"id":"b29745ed-9098-46fe-910c-5b80b201615e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13bf337a b29745ed-9098-46fe-910c-5b80b201615e /tidb/cdc/default/default/upstream/7362129765043607146 {"id":7362129765043607146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b29745ed-9098-46fe-910c-5b80b201615e {"id":"b29745ed-9098-46fe-910c-5b80b201615e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13bf337a b29745ed-9098-46fe-910c-5b80b201615e /tidb/cdc/default/default/upstream/7362129765043607146 {"id":7362129765043607146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 9c5a81f8-e50b-48f0-976a-39a921182373 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=9c5a81f8-e50b-48f0-976a-39a921182373 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 9c5a81f8-e50b-48f0-976a-39a921182373 -s + info='{ "upstream_id": 7362129765043607146, "namespace": "default", "id": "9c5a81f8-e50b-48f0-976a-39a921182373", "state": "normal", "checkpoint_tso": 449348770676604930, "checkpoint_time": "2024-04-26 19:05:26.812", "error": null }' + echo '{ "upstream_id": 7362129765043607146, "namespace": "default", "id": "9c5a81f8-e50b-48f0-976a-39a921182373", "state": "normal", "checkpoint_tso": 449348770676604930, "checkpoint_time": "2024-04-26 19:05:26.812", "error": null }' { "upstream_id": 7362129765043607146, "namespace": "default", "id": "9c5a81f8-e50b-48f0-976a-39a921182373", "state": "normal", "checkpoint_tso": 449348770676604930, "checkpoint_time": "2024-04-26 19:05:26.812", "error": null } ++ echo '{' '"upstream_id":' 7362129765043607146, '"namespace":' '"default",' '"id":' '"9c5a81f8-e50b-48f0-976a-39a921182373",' '"state":' '"normal",' '"checkpoint_tso":' 449348770676604930, '"checkpoint_time":' '"2024-04-26' '19:05:26.812",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362129765043607146, '"namespace":' '"default",' '"id":' '"9c5a81f8-e50b-48f0-976a-39a921182373",' '"state":' '"normal",' '"checkpoint_tso":' 449348770676604930, '"checkpoint_time":' '"2024-04-26' '19:05:26.812",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:05:32 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Resume changefeed with checkpointTs1 449348753834901505 table test.table3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.table3 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685170cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36321, start at 2024-04-26 19:04:53.067934959 +0800 CST m=+5.353551115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:53.073 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:53.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:53.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table test.table3 exists table test.table4 not exists for 1-th check, retry later table test.table4 exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:05:44 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.4652446526.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851ebcc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32543, start at 2024-04-26 19:05:00.937291156 +0800 CST m=+4.383856077 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.915 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.915 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6851eb000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:32632, start at 2024-04-26 19:05:00.880259273 +0800 CST m=+4.257608988 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:00.886 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:00.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:00.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:05:47 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.3663736639.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e70656c6-5871-4799-876b-aaafbb4e2c4d {"id":"e70656c6-5871-4799-876b-aaafbb4e2c4d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21f4b e70656c6-5871-4799-876b-aaafbb4e2c4d /tidb/cdc/default/default/changefeed/info/59b227e1-34ea-436a-bdcf-a1930b796d9f {"upstream-id":7362129753739812378,"namespace":"default","changefeed-id":"59b227e1-34ea-436a-bdcf-a1930b796d9f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:04:22.484783312+08:00","start-ts":449348753834901505,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-04-26T19:05:45.167001472+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348775488258049} /tidb/cdc/default/default/changefeed/status/59b227e1-34ea-436a-bdcf-a1930b796d9f {"checkpoint-ts":449348770048507920,"min-table-barrier-ts":449348770074722339,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e70656c6-5871-4799-876b-aaafbb4e2c4d {"id":"e70656c6-5871-4799-876b-aaafbb4e2c4d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21f4b e70656c6-5871-4799-876b-aaafbb4e2c4d /tidb/cdc/default/default/changefeed/info/59b227e1-34ea-436a-bdcf-a1930b796d9f {"upstream-id":7362129753739812378,"namespace":"default","changefeed-id":"59b227e1-34ea-436a-bdcf-a1930b796d9f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:04:22.484783312+08:00","start-ts":449348753834901505,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-04-26T19:05:45.167001472+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348775488258049} /tidb/cdc/default/default/changefeed/status/59b227e1-34ea-436a-bdcf-a1930b796d9f {"checkpoint-ts":449348770048507920,"min-table-barrier-ts":449348770074722339,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e70656c6-5871-4799-876b-aaafbb4e2c4d {"id":"e70656c6-5871-4799-876b-aaafbb4e2c4d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13b21f4b e70656c6-5871-4799-876b-aaafbb4e2c4d /tidb/cdc/default/default/changefeed/info/59b227e1-34ea-436a-bdcf-a1930b796d9f {"upstream-id":7362129753739812378,"namespace":"default","changefeed-id":"59b227e1-34ea-436a-bdcf-a1930b796d9f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:04:22.484783312+08:00","start-ts":449348753834901505,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"failed","error":{"time":"2024-04-26T19:05:45.167001472+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348775488258049} /tidb/cdc/default/default/changefeed/status/59b227e1-34ea-436a-bdcf-a1930b796d9f {"checkpoint-ts":449348770048507920,"min-table-barrier-ts":449348770074722339,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362129753739812378 {"id":7362129753739812378,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 59b227e1-34ea-436a-bdcf-a1930b796d9f failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=59b227e1-34ea-436a-bdcf-a1930b796d9f + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 59b227e1-34ea-436a-bdcf-a1930b796d9f -s + info='{ "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "failed", "checkpoint_tso": 449348770048507920, "checkpoint_time": "2024-04-26 19:05:24.416", "error": { "time": "2024-04-26T19:05:45.167001472+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "failed", "checkpoint_tso": 449348770048507920, "checkpoint_time": "2024-04-26 19:05:24.416", "error": { "time": "2024-04-26T19:05:45.167001472+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "failed", "checkpoint_tso": 449348770048507920, "checkpoint_time": "2024-04-26 19:05:24.416", "error": { "time": "2024-04-26T19:05:45.167001472+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7362129753739812378, '"namespace":' '"default",' '"id":' '"59b227e1-34ea-436a-bdcf-a1930b796d9f",' '"state":' '"failed",' '"checkpoint_tso":' 449348770048507920, '"checkpoint_time":' '"2024-04-26' '19:05:24.416",' '"error":' '{' '"time":' '"2024-04-26T19:05:45.167001472+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7362129753739812378, '"namespace":' '"default",' '"id":' '"59b227e1-34ea-436a-bdcf-a1930b796d9f",' '"state":' '"failed",' '"checkpoint_tso":' 449348770048507920, '"checkpoint_time":' '"2024-04-26' '19:05:24.416",' '"error":' '{' '"time":' '"2024-04-26T19:05:45.167001472+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully check_changefeed_state http://127.0.0.1:2379 59b227e1-34ea-436a-bdcf-a1930b796d9f normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=59b227e1-34ea-436a-bdcf-a1930b796d9f + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 59b227e1-34ea-436a-bdcf-a1930b796d9f -s + info='{ "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "normal", "checkpoint_tso": 449348776314011653, "checkpoint_time": "2024-04-26 19:05:48.317", "error": null }' + echo '{ "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "normal", "checkpoint_tso": 449348776314011653, "checkpoint_time": "2024-04-26 19:05:48.317", "error": null }' { "upstream_id": 7362129753739812378, "namespace": "default", "id": "59b227e1-34ea-436a-bdcf-a1930b796d9f", "state": "normal", "checkpoint_tso": 449348776314011653, "checkpoint_time": "2024-04-26 19:05:48.317", "error": null } ++ echo '{' '"upstream_id":' 7362129753739812378, '"namespace":' '"default",' '"id":' '"59b227e1-34ea-436a-bdcf-a1930b796d9f",' '"state":' '"normal",' '"checkpoint_tso":' 449348776314011653, '"checkpoint_time":' '"2024-04-26' '19:05:48.317",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362129753739812378, '"namespace":' '"default",' '"id":' '"59b227e1-34ea-436a-bdcf-a1930b796d9f",' '"state":' '"normal",' '"checkpoint_tso":' 449348776314011653, '"checkpoint_time":' '"2024-04-26' '19:05:48.317",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table test1.table1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:05:51 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/544bb4fa-ff33-416d-bb62-23198f9d06aa {"id":"544bb4fa-ff33-416d-bb62-23198f9d06aa","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15089988 544bb4fa-ff33-416d-bb62-23198f9d06aa /tidb/cdc/default/default/upstream/7362130116269711190 {"id":7362130116269711190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/544bb4fa-ff33-416d-bb62-23198f9d06aa {"id":"544bb4fa-ff33-416d-bb62-23198f9d06aa","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15089988 544bb4fa-ff33-416d-bb62-23198f9d06aa /tidb/cdc/default/default/upstream/7362130116269711190 {"id":7362130116269711190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/544bb4fa-ff33-416d-bb62-23198f9d06aa {"id":"544bb4fa-ff33-416d-bb62-23198f9d06aa","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15089988 544bb4fa-ff33-416d-bb62-23198f9d06aa /tidb/cdc/default/default/upstream/7362130116269711190 {"id":7362130116269711190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.36692.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml table test1.table1 not exists for 2-th check, retry later Create changefeed successfully! ID: 0195c936-dff3-4da9-b960-7f98747b34ca Info: {"upstream_id":7362130116269711190,"namespace":"default","id":"0195c936-dff3-4da9-b960-7f98747b34ca","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:05:51.375137104+08:00","start_ts":449348777092579331,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348777092579331,"checkpoint_ts":449348777092579331,"checkpoint_time":"2024-04-26 19:05:51.287"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test1.table1 exists table test1.table2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685514780016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40799, start at 2024-04-26 19:05:52.691748865 +0800 CST m=+4.291001148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.670 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.670 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685381f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38450, start at 2024-04-26 19:05:26.924531603 +0800 CST m=+4.276848453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:26.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:26.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:26.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table test1.table2 exists check diff successfully Starting CDC state checker... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:05:56 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.user"="root" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.db"="sink_retry" "dotransactions"="false" "insertproportion"="0" "workload"="core" "requestdistribution"="uniform" "recordcount"="10" "mysql.port"="4000" "threadcount"="2" "readproportion"="0" "operationcount"="0" "scanproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 4.522212ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3216.7, Avg(us): 817, Min(us): 551, Max(us): 1804, 95th(us): 2000, 99th(us): 2000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation_without_old_value Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685169240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36233, start at 2024-04-26 19:04:52.580507957 +0800 CST m=+4.962817615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:52.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:52.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:52.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685170cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:36321, start at 2024-04-26 19:04:53.067934959 +0800 CST m=+5.353551115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:06:53.073 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:04:53.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:54:53.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:06:06 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.4039940401.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:09 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db96fec-e7dc-43af-a602-8c1f9714f5c3 {"id":"8db96fec-e7dc-43af-a602-8c1f9714f5c3","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15335596 8db96fec-e7dc-43af-a602-8c1f9714f5c3 /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db96fec-e7dc-43af-a602-8c1f9714f5c3 {"id":"8db96fec-e7dc-43af-a602-8c1f9714f5c3","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15335596 8db96fec-e7dc-43af-a602-8c1f9714f5c3 /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db96fec-e7dc-43af-a602-8c1f9714f5c3 {"id":"8db96fec-e7dc-43af-a602-8c1f9714f5c3","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15335596 8db96fec-e7dc-43af-a602-8c1f9714f5c3 /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Fri Apr 26 19:06:01 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.4940249404.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:04 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6c5cff-ea28-430e-881a-9c85110822de {"id":"cc6c5cff-ea28-430e-881a-9c85110822de","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1456a409 cc6c5cff-ea28-430e-881a-9c85110822de /tidb/cdc/default/default/upstream/7362129905643220423 {"id":7362129905643220423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6c5cff-ea28-430e-881a-9c85110822de {"id":"cc6c5cff-ea28-430e-881a-9c85110822de","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1456a409 cc6c5cff-ea28-430e-881a-9c85110822de /tidb/cdc/default/default/upstream/7362129905643220423 {"id":7362129905643220423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6c5cff-ea28-430e-881a-9c85110822de {"id":"cc6c5cff-ea28-430e-881a-9c85110822de","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1456a409 cc6c5cff-ea28-430e-881a-9c85110822de /tidb/cdc/default/default/upstream/7362129905643220423 {"id":7362129905643220423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.49462.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 76188d57-7c1b-4ee2-905f-08d4b01fd5a1 Info: {"upstream_id":7362129905643220423,"namespace":"default","id":"76188d57-7c1b-4ee2-905f-08d4b01fd5a1","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-04-26T19:06:04.794086201+08:00","start_ts":449348780600328196,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348780600328196,"checkpoint_ts":449348780600328196,"checkpoint_time":"2024-04-26 19:06:04.668"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 [2024/04/26 19:06:05.654 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:05.655 +08:00] [INFO] [case.go:444] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:05.656 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq0] [2024/04/26 19:06:06.173 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:06.182 +08:00] [INFO] [case.go:364] ["insert 100 takes 9.638158ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:06.183 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq1] [2024/04/26 19:06:06.695 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:06.703 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.154962ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:06.704 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq2] [2024/04/26 19:06:07.215 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:07.224 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.688484ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:07.225 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq3] [2024/04/26 19:06:07.734 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:07.742 +08:00] [INFO] [case.go:364] ["insert 100 takes 7.938778ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:07.742 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts_seq4] [2024/04/26 19:06:08.252 +08:00] [INFO] [case.go:137] ["batch insert sql"] [sql="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:08.261 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.839123ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/04/26 19:06:08.262 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts0] [2024/04/26 19:06:08.780 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.254202ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/04/26 19:06:08.780 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts1] [2024/04/26 19:06:09.299 +08:00] [INFO] [case.go:364] ["insert 100 takes 9.084291ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/04/26 19:06:09.300 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts2] [2024/04/26 19:06:09.824 +08:00] [INFO] [case.go:364] ["insert 100 takes 10.320578ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/04/26 19:06:09.824 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts3] [2024/04/26 19:06:10.344 +08:00] [INFO] [case.go:364] ["insert 100 takes 8.523227ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/04/26 19:06:10.345 +08:00] [INFO] [case.go:382] ["drop tables"] [table=accounts4] [2024/04/26 19:06:10.864 +08:00] [INFO] [case.go:364] ["insert 100 takes 9.568936ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:13.375 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685383900004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38356, start at 2024-04-26 19:05:27.016161422 +0800 CST m=+4.432314982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:27.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:27.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:27.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685381f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--hf26k, pid:38450, start at 2024-04-26 19:05:26.924531603 +0800 CST m=+4.276848453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:26.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:26.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:26.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:06:14 CST 2024] <<<<<< START cdc server in new_ci_collation_without_old_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation_without_old_value.4254842550.out server --log-file /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation_without_old_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:15.386 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:12.945 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m35.298570392s] [2024/04/26 19:06:12.948 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [Fri Apr 26 19:06:05 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.3582935831.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:08 GMT < Content-Length: 604 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/8c7f5e36-a6f1-463f-93bf-95d22b96a2dc {"id":"8c7f5e36-a6f1-463f-93bf-95d22b96a2dc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f1a146b830e 8c7f5e36-a6f1-463f-93bf-95d22b96a2dc /tidb/cdc/test1/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/8c7f5e36-a6f1-463f-93bf-95d22b96a2dc {"id":"8c7f5e36-a6f1-463f-93bf-95d22b96a2dc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f1a146b830e 8c7f5e36-a6f1-463f-93bf-95d22b96a2dc /tidb/cdc/test1/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/8c7f5e36-a6f1-463f-93bf-95d22b96a2dc {"id":"8c7f5e36-a6f1-463f-93bf-95d22b96a2dc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f1a146b830e 8c7f5e36-a6f1-463f-93bf-95d22b96a2dc /tidb/cdc/test1/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:06:08 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.3588335885.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:11 GMT < Content-Length: 604 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/70e283df-2045-4fe8-b0d4-897cacf1aba3 {"id":"70e283df-2045-4fe8-b0d4-897cacf1aba3","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f1a146b832b 70e283df-2045-4fe8-b0d4-897cacf1aba3 /tidb/cdc/test2/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/70e283df-2045-4fe8-b0d4-897cacf1aba3 {"id":"70e283df-2045-4fe8-b0d4-897cacf1aba3","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f1a146b832b 70e283df-2045-4fe8-b0d4-897cacf1aba3 /tidb/cdc/test2/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/70e283df-2045-4fe8-b0d4-897cacf1aba3 {"id":"70e283df-2045-4fe8-b0d4-897cacf1aba3","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f1a146b832b 70e283df-2045-4fe8-b0d4-897cacf1aba3 /tidb/cdc/test2/default/upstream/7362129957159986670 {"id":7362129957159986670,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.35943.out cli changefeed create --start-ts=449348765979508737 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: a7cc9243-2ee0-41be-816a-31b615a99194 Info: {"upstream_id":7362129957159986670,"namespace":"default","id":"a7cc9243-2ee0-41be-816a-31b615a99194","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:06:12.352110647+08:00","start_ts":449348765979508737,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348765979508737,"checkpoint_ts":449348765979508737,"checkpoint_time":"2024-04-26 19:05:08.894"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.35977.out cli changefeed create --start-ts=449348765979508737 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml Create changefeed successfully! ID: ccb5186a-f899-416e-8dd4-741e6f50dcbd Info: {"upstream_id":7362129957159986670,"namespace":"default","id":"ccb5186a-f899-416e-8dd4-741e6f50dcbd","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:06:13.047533025+08:00","start_ts":449348765979508737,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348765979508737,"checkpoint_ts":449348765979508737,"checkpoint_time":"2024-04-26 19:05:08.894"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.multi_cdc1 not exists for 1-th check, retry later table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685514780016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40799, start at 2024-04-26 19:05:52.691748865 +0800 CST m=+4.291001148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.670 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.670 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 19:06:17.397 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:18 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/126e09ac-c308-42db-9b45-eb6622c80fbc {"id":"126e09ac-c308-42db-9b45-eb6622c80fbc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1573a38d 126e09ac-c308-42db-9b45-eb6622c80fbc /tidb/cdc/default/default/upstream/7362130219147790775 {"id":7362130219147790775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/126e09ac-c308-42db-9b45-eb6622c80fbc {"id":"126e09ac-c308-42db-9b45-eb6622c80fbc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1573a38d 126e09ac-c308-42db-9b45-eb6622c80fbc /tidb/cdc/default/default/upstream/7362130219147790775 {"id":7362130219147790775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/126e09ac-c308-42db-9b45-eb6622c80fbc {"id":"126e09ac-c308-42db-9b45-eb6622c80fbc","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1573a38d 126e09ac-c308-42db-9b45-eb6622c80fbc /tidb/cdc/default/default/upstream/7362130219147790775 {"id":7362130219147790775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 87a9bdc7-dd81-4dd3-9d6d-226a61f38a99 Info: {"upstream_id":7362130219147790775,"namespace":"default","id":"87a9bdc7-dd81-4dd3-9d6d-226a61f38a99","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-04-26T19:06:18.156172674+08:00","start_ts":449348783280750593,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348783280750593,"checkpoint_ts":449348783280750593,"checkpoint_time":"2024-04-26 19:06:14.893"} cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:06:18 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685691c0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47922, start at 2024-04-26 19:06:17.083837439 +0800 CST m=+4.471866796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.072 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.072 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log [2024/04/26 19:06:18.985 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:06:19 CST 2024] <<<<<< run test case cdc success! >>>>>> [2024/04/26 19:06:19.407 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] Starting CDC state checker... [2024/04/26 19:06:21.416 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Fri Apr 26 19:06:21 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Fri Apr 26 19:06:21 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml [2024/04/26 19:06:23.426 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 [2024/04/26 19:06:25.439 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:06:27.449 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/multi_changefeed/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 30626 6.3 0.3 3164456 1443156 ? Sl 19:05 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 30627 6.6 0.3 3164452 1446080 ? Sl 19:05 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 30628 6.8 0.3 3155236 1453516 ? Sl 19:05 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 30630 6.5 0.3 3164456 1442860 ? Sl 19:05 0:04 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6461 0 --:--:-- --:--:-- --:--:-- 6567 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:06:27.739","puller_resolved_ts":"2024-04-26 19:06:27.788","last_synced_ts":"2024-04-26 19:06:21.888","now_ts":"2024-04-26 19:06:28.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:06:27.739","puller_resolved_ts":"2024-04-26' '19:06:27.788","last_synced_ts":"2024-04-26' '19:06:21.888","now_ts":"2024-04-26' '19:06:28.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:06:27.739","puller_resolved_ts":"2024-04-26' '19:06:27.788","last_synced_ts":"2024-04-26' '19:06:21.888","now_ts":"2024-04-26' '19:06:28.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:06:29.459 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:29.468 +08:00] [INFO] [case.go:496] ["all tables synced"] [endTs="2024-04-26 19:06:27"] check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table sink_retry.finish_mark_2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:06:29 CST 2024] <<<<<< run test case sink_retry success! >>>>>> check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:06:32.804 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark1000] [2024/04/26 19:06:32.804 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark1000] Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Fri Apr 26 19:06:34 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> Starting Upstream TiDB... [2024/04/26 19:06:34.804 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:34.817 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark1000] [2024/04/26 19:06:34.817 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:33"] [2024/04/26 19:06:34.820 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:34.902 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:34.909 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:34.914 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685515f0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40702, start at 2024-04-26 19:05:52.778423096 +0800 CST m=+4.432473129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.764 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.764 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685514780016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:40799, start at 2024-04-26 19:05:52.691748865 +0800 CST m=+4.291001148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:07:52.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:05:52.670 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:55:52.670 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:06:35 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.4485044852.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:06:36.326 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark2000] [2024/04/26 19:06:36.326 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark2000] Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:06:38.326 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:38.338 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark2000] [2024/04/26 19:06:38.338 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:37"] [2024/04/26 19:06:38.340 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:38.343 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:38.413 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:38.420 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:38 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 {"id":"6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15c9118e 6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 /tidb/cdc/default/default/upstream/7362130338739508119 {"id":7362130338739508119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 {"id":"6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15c9118e 6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 /tidb/cdc/default/default/upstream/7362130338739508119 {"id":7362130338739508119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 {"id":"6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a15c9118e 6250a3a9-b45d-4bbd-9104-aa2d2b0b97c5 /tidb/cdc/default/default/upstream/7362130338739508119 {"id":7362130338739508119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:39.305 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark3000] [2024/04/26 19:06:39.305 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark3000] Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/sorter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:37025, start at 2024-04-26 19:06:39.633407323 +0800 CST m=+4.692442498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.647 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.647 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_changefeed/tiflash/log/error.log ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:41.305 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685691c0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47922, start at 2024-04-26 19:06:17.083837439 +0800 CST m=+4.471866796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.072 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.072 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [2024/04/26 19:06:43.320 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:43.403 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark3000] [2024/04/26 19:06:43.403 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:41"] [2024/04/26 19:06:43.405 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:43.408 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:43.412 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:43.416 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 19:06:44.431 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark4000] [2024/04/26 19:06:44.431 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark4000] Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685832700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37229, start at 2024-04-26 19:06:43.780062354 +0800 CST m=+4.309781895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:06:46.432 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:46.513 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark4000] [2024/04/26 19:06:46.513 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:45"] [2024/04/26 19:06:46.599 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:46.603 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:46.614 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:46.621 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Starting CDC state checker... [2024/04/26 19:06:47.712 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark5000] [2024/04/26 19:06:47.712 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark5000] Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:06:49.713 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:49.727 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark5000] [2024/04/26 19:06:49.727 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:48"] [2024/04/26 19:06:49.732 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:49.802 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:49.812 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:49.819 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:06:51.907 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark6000] [2024/04/26 19:06:51.907 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark6000] start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:06:53.908 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:54.013 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark6000] [2024/04/26 19:06:54.013 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:52"] [2024/04/26 19:06:54.016 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:54.021 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:54.026 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:54.031 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858c0a40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42889, start at 2024-04-26 19:06:52.84808282 +0800 CST m=+4.674987855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sorter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sorter/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:06:55.102 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark7000] [2024/04/26 19:06:55.102 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark7000] ***************** properties ***************** "operationcount"="0" "requestdistribution"="uniform" "insertproportion"="0" "fieldcount"="3" "mysql.host"="127.0.0.1" "workload"="core" "recordcount"="50" "scanproportion"="0" "updateproportion"="0" "readproportion"="0" "dotransactions"="false" "mysql.user"="root" "threadcount"="10" "mysql.db"="consistent_replicate_ddl" "readallfields"="true" "mysql.port"="4000" ********************************************** Run finished, takes 11.566355ms INSERT - Takes(s): 0.0, Count: 50, OPS: 8652.3, Avg(us): 2130, Min(us): 843, Max(us): 5773, 95th(us): 6000, 99th(us): 6000 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... [2024/04/26 19:06:57.103 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:06:57.199 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark7000] [2024/04/26 19:06:57.199 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:56"] [2024/04/26 19:06:57.205 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:57.208 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:06:57.213 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:06:57.219 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [2024/04/26 19:06:57.802 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark8000] [2024/04/26 19:06:57.802 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark8000] table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later table event_filter.t_alter exists table event_filter.finish_mark exists check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:06:59 CST 2024] <<<<<< run test case event_filter success! >>>>>> [2024/04/26 19:06:59.802 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685692700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47830, start at 2024-04-26 19:06:17.140222825 +0800 CST m=+4.583693217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.116 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.116 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685691c0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:47922, start at 2024-04-26 19:06:17.083837439 +0800 CST m=+4.471866796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:17.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:17.072 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:17.072 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:07:00 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.5203952041.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:07:01.416 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark9000] [2024/04/26 19:07:01.416 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark9000] [2024/04/26 19:07:01.814 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:01.901 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark8000] [2024/04/26 19:07:01.901 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:06:59"] [2024/04/26 19:07:01.903 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:01.906 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:01.911 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:01.915 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937400006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44497, start at 2024-04-26 19:07:00.442326159 +0800 CST m=+4.446195842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.432 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.432 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:07:03.417 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:03.512 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark9000] [2024/04/26 19:07:03.512 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:02"] [2024/04/26 19:07:03.515 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:03.602 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:03.606 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:03.610 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:07:03 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e4da4d-6bd2-4f3e-af29-16fe15cf3526 {"id":"45e4da4d-6bd2-4f3e-af29-16fe15cf3526","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a162a8f8e 45e4da4d-6bd2-4f3e-af29-16fe15cf3526 /tidb/cdc/default/default/upstream/7362130442959147243 {"id":7362130442959147243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e4da4d-6bd2-4f3e-af29-16fe15cf3526 {"id":"45e4da4d-6bd2-4f3e-af29-16fe15cf3526","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a162a8f8e 45e4da4d-6bd2-4f3e-af29-16fe15cf3526 /tidb/cdc/default/default/upstream/7362130442959147243 {"id":7362130442959147243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45e4da4d-6bd2-4f3e-af29-16fe15cf3526 {"id":"45e4da4d-6bd2-4f3e-af29-16fe15cf3526","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a162a8f8e 45e4da4d-6bd2-4f3e-af29-16fe15cf3526 /tidb/cdc/default/default/upstream/7362130442959147243 {"id":7362130442959147243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 03bd58b1-589d-48b7-be60-4b9d9322aefc Info: {"upstream_id":7362130442959147243,"namespace":"default","id":"03bd58b1-589d-48b7-be60-4b9d9322aefc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:07:04.081978735+08:00","start_ts":449348795306868740,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348795306868740,"checkpoint_ts":449348795306868740,"checkpoint_time":"2024-04-26 19:07:00.769"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:37025, start at 2024-04-26 19:06:39.633407323 +0800 CST m=+4.692442498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.647 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.647 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:07:05.103 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark10000] [2024/04/26 19:07:05.103 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark10000] # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... pass check, checkpoint tso not forward after 10s run task successfully wait process 29300 exit for 1-th time... wait process 29300 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (29300) - No such process wait process 29300 exit for 3-th time... process 29300 already exit [Fri Apr 26 19:06:52 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.2994329945.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:06:55 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc000645320 TaskPositions:map[9af70fee-9262-4969-ab56-b785916986a4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 84, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 86, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 88, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af70fee-9262-4969-ab56-b785916986a4 {"id":"9af70fee-9262-4969-ab56-b785916986a4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04dac 9af70fee-9262-4969-ab56-b785916986a4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770464006148,"min-table-barrier-ts":449348794030489603,"admin-job-type":0} /tidb/cdc/default/default/task/position/9af70fee-9262-4969-ab56-b785916986a4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc000645320 TaskPositions:map[9af70fee-9262-4969-ab56-b785916986a4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 84, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 86, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 88, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af70fee-9262-4969-ab56-b785916986a4 {"id":"9af70fee-9262-4969-ab56-b785916986a4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04dac 9af70fee-9262-4969-ab56-b785916986a4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770464006148,"min-table-barrier-ts":449348794030489603,"admin-job-type":0} /tidb/cdc/default/default/task/position/9af70fee-9262-4969-ab56-b785916986a4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {ClusterID:default ID:default/ddl-only-block-related-table Info:{"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} Status:0xc000645320 TaskPositions:map[9af70fee-9262-4969-ab56-b785916986a4:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 84, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 86, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing tableID: 88, resolvedTs: 449348770464006148, checkpointTs: 449348770464006148, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af70fee-9262-4969-ab56-b785916986a4 {"id":"9af70fee-9262-4969-ab56-b785916986a4","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a13a04dac 9af70fee-9262-4969-ab56-b785916986a4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362129734779594518,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-04-26T19:04:18.197043504+08:00","start-ts":449348752651059206,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348752677535751} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449348770464006148,"min-table-barrier-ts":449348794030489603,"admin-job-type":0} /tidb/cdc/default/default/task/position/9af70fee-9262-4969-ab56-b785916986a4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362129734779594518 {"id":7362129734779594518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449348770464006140->449348794292633600 checkpoint: 449348770464006140->449348794554777600 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:06:59 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later [2024/04/26 19:07:07.103 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:07.120 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark10000] [2024/04/26 19:07:07.120 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:06"] [2024/04/26 19:07:07.122 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:07.203 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:07.207 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:07.212 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:07.316 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark11000] [2024/04/26 19:07:07.316 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark11000] start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685832700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37229, start at 2024-04-26 19:06:43.780062354 +0800 CST m=+4.309781895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:07:09.316 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:10.263 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark12000] [2024/04/26 19:07:10.263 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark12000] table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... [2024/04/26 19:07:11.330 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:11.342 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark11000] [2024/04/26 19:07:11.342 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:09"] Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:07:11.400 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:11.407 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:11.413 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:11.417 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:07:12.264 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] table consistent_replicate_ddl.usertable_bak exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:07:13.607 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark13000] [2024/04/26 19:07:13.607 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark13000] [2024/04/26 19:07:14.299 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:14.315 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark12000] [2024/04/26 19:07:14.315 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:13"] [2024/04/26 19:07:14.327 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:14.331 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:14.335 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:14.401 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] start tidb cluster in /tmp/tidb_cdc_test/multi_changefeed Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table new_ci_collation_without_old_value_test.t1 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:07:15.608 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] table new_ci_collation_without_old_value_test.t1 exists table new_ci_collation_without_old_value_test.t2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858c0a40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42889, start at 2024-04-26 19:06:52.84808282 +0800 CST m=+4.674987855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:07:17.622 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:17.633 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark13000] [2024/04/26 19:07:17.633 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:16"] [2024/04/26 19:07:17.714 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] table new_ci_collation_without_old_value_test.t2 exists table new_ci_collation_without_old_value_test.t3 not exists for 1-th check, retry later [2024/04/26 19:07:17.718 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:17.724 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:17.728 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:17.823 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark14000] [2024/04/26 19:07:17.823 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark14000] cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:07:17 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.4067440676.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:07:19.823 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:07:19.916 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark14000] [2024/04/26 19:07:19.917 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:18"] [2024/04/26 19:07:19.922 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:19.926 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:20.015 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:20.023 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:20.115 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] table new_ci_collation_without_old_value_test.t3 exists table new_ci_collation_without_old_value_test.t4 not exists for 1-th check, retry later [2024/04/26 19:07:20.214 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:20.308 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:20.337 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] Verifying downstream PD is started... [2024/04/26 19:07:20.911 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:07:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/08fee803-cf88-4d72-906d-e01ef2faddea {ClusterID:default ID:default/08fee803-cf88-4d72-906d-e01ef2faddea Info:{"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} Status:0xc0039e92a8 TaskPositions:map[89158f56-d2a3-4f77-b0ff-58424e97d0b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 84, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 88, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 90, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 86, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89158f56-d2a3-4f77-b0ff-58424e97d0b5 {"id":"89158f56-d2a3-4f77-b0ff-58424e97d0b5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1533570c 89158f56-d2a3-4f77-b0ff-58424e97d0b5 /tidb/cdc/default/default/changefeed/info/08fee803-cf88-4d72-906d-e01ef2faddea {"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} /tidb/cdc/default/default/changefeed/status/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":449348800238845957,"min-table-barrier-ts":449348800500989958,"admin-job-type":0} /tidb/cdc/default/default/task/position/89158f56-d2a3-4f77-b0ff-58424e97d0b5/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/08fee803-cf88-4d72-906d-e01ef2faddea {ClusterID:default ID:default/08fee803-cf88-4d72-906d-e01ef2faddea Info:{"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} Status:0xc0039e92a8 TaskPositions:map[89158f56-d2a3-4f77-b0ff-58424e97d0b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 84, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 88, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 90, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 86, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89158f56-d2a3-4f77-b0ff-58424e97d0b5 {"id":"89158f56-d2a3-4f77-b0ff-58424e97d0b5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1533570c 89158f56-d2a3-4f77-b0ff-58424e97d0b5 /tidb/cdc/default/default/changefeed/info/08fee803-cf88-4d72-906d-e01ef2faddea {"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} /tidb/cdc/default/default/changefeed/status/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":449348800238845957,"min-table-barrier-ts":449348800500989958,"admin-job-type":0} /tidb/cdc/default/default/task/position/89158f56-d2a3-4f77-b0ff-58424e97d0b5/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/08fee803-cf88-4d72-906d-e01ef2faddea {ClusterID:default ID:default/08fee803-cf88-4d72-906d-e01ef2faddea Info:{"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} Status:0xc0039e92a8 TaskPositions:map[89158f56-d2a3-4f77-b0ff-58424e97d0b5:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 82, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 84, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 88, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 90, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating tableID: 86, resolvedTs: 449348800238845958, checkpointTs: 449348800238845957, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89158f56-d2a3-4f77-b0ff-58424e97d0b5 {"id":"89158f56-d2a3-4f77-b0ff-58424e97d0b5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1533570c 89158f56-d2a3-4f77-b0ff-58424e97d0b5 /tidb/cdc/default/default/changefeed/info/08fee803-cf88-4d72-906d-e01ef2faddea {"upstream-id":7362130071674132119,"namespace":"default","changefeed-id":"08fee803-cf88-4d72-906d-e01ef2faddea","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:06:09.26579957+08:00","start-ts":449348781783908356,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348781797015558} /tidb/cdc/default/default/changefeed/status/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":449348800238845957,"min-table-barrier-ts":449348800500989958,"admin-job-type":0} /tidb/cdc/default/default/task/position/89158f56-d2a3-4f77-b0ff-58424e97d0b5/08fee803-cf88-4d72-906d-e01ef2faddea {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362130071674132119 {"id":7362130071674132119,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:07:21.535 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark15000] [2024/04/26 19:07:21.535 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark15000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_without_old_value_test.t4 exists table new_ci_collation_without_old_value_test.t5 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:07:22.804 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:23.002 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1f80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:36964, start at 2024-04-26 19:06:39.654062506 +0800 CST m=+4.790158871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6857f1b00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kdjsj, pid:37025, start at 2024-04-26 19:06:39.633407323 +0800 CST m=+4.692442498 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:39.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:39.647 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:39.647 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:07:23 CST 2024] <<<<<< START cdc server in multi_changefeed case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_changefeed.4089340895.out server --log-file /tmp/tidb_cdc_test/multi_changefeed/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_changefeed/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:07:23.536 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:23.627 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark15000] [2024/04/26 19:07:23.627 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:22"] [2024/04/26 19:07:23.629 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:23.704 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:23.716 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:23.724 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a864c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:31041, start at 2024-04-26 19:07:21.907781121 +0800 CST m=+4.464500185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log table new_ci_collation_without_old_value_test.t5 exists check diff failed 1-th time, retry later Starting Upstream TiDB... [2024/04/26 19:07:24.207 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:24.326 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark16000] [2024/04/26 19:07:24.326 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark16000] Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a8fe80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37820, start at 2024-04-26 19:07:22.519412729 +0800 CST m=+4.611188916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937400006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44497, start at 2024-04-26 19:07:00.442326159 +0800 CST m=+4.446195842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.432 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.432 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... *************************** 1. row *************************** count(distinct region_id): 2 split_and_random_merge scale: 80 Starting CDC state checker... Starting CDC state checker... check diff successfully check diff failed 1-th time, retry later # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sorter Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... [2024/04/26 19:07:26.326 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:26.399 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark16000] [2024/04/26 19:07:26.401 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:25"] [2024/04/26 19:07:26.409 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:26.416 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:26.435 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:26.511 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:26.603 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:26.702 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685831d00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37133, start at 2024-04-26 19:06:43.741502476 +0800 CST m=+4.327003158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.749 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.749 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685832700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:37229, start at 2024-04-26 19:06:43.780062354 +0800 CST m=+4.309781895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:43.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:43.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:43.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.41378.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:07:26 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2791a80-0cba-4847-a26a-2ab87f31e288 {"id":"b2791a80-0cba-4847-a26a-2ab87f31e288","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1681a78e b2791a80-0cba-4847-a26a-2ab87f31e288 /tidb/cdc/default/default/upstream/7362130535144974644 {"id":7362130535144974644,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2791a80-0cba-4847-a26a-2ab87f31e288 {"id":"b2791a80-0cba-4847-a26a-2ab87f31e288","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1681a78e b2791a80-0cba-4847-a26a-2ab87f31e288 /tidb/cdc/default/default/upstream/7362130535144974644 {"id":7362130535144974644,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2791a80-0cba-4847-a26a-2ab87f31e288 {"id":"b2791a80-0cba-4847-a26a-2ab87f31e288","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1681a78e b2791a80-0cba-4847-a26a-2ab87f31e288 /tidb/cdc/default/default/upstream/7362130535144974644 {"id":7362130535144974644,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: old-value-cf Info: {"upstream_id":7362130535144974644,"namespace":"default","id":"old-value-cf","sink_uri":"blackhole:","create_time":"2024-04-26T19:07:26.677718754+08:00","start_ts":449348802082504707,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348802082504707,"checkpoint_ts":449348802082504707,"checkpoint_time":"2024-04-26 19:07:26.616"} Create changefeed successfully! ID: no-old-value-cf Info: {"upstream_id":7362130535144974644,"namespace":"default","id":"no-old-value-cf","sink_uri":"blackhole:","create_time":"2024-04-26T19:07:26.877260322+08:00","start_ts":449348802121564162,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348802121564162,"checkpoint_ts":449348802121564162,"checkpoint_time":"2024-04-26 19:07:26.765"} + set +x + tso='449348802073591809 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348802073591809 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/04/26 19:07:26.901 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:27.306 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:27.530 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:27.922 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark17000] [2024/04/26 19:07:27.922 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark17000] [2024/04/26 19:07:28.405 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] check diff successfully wait process cdc.test exit for 1-th time... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:07:29 CST 2024] <<<<<< run test case new_ci_collation_without_old_value success! >>>>>> [2024/04/26 19:07:29.907 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts4 expect 100000, but got 99828"] [errorVerbose="verify balance failed, accounts4 expect 100000, but got 99828\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:29.922 +08:00] [INFO] [case.go:633] ["try get end ts"] [query="SELECT END_TIME FROM information_schema.ddl_jobs WHERE table_name = ?"] [2024/04/26 19:07:29.933 +08:00] [INFO] [case.go:573] ["ddl synced"] [table=finishmark17000] [2024/04/26 19:07:29.933 +08:00] [INFO] [case.go:579] ["downstream sync success"] [endTs="2024-04-26 19:07:28"] [2024/04/26 19:07:30.015 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:30.018 +08:00] [INFO] [case.go:185] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/04/26 19:07:30.028 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/04/26 19:07:30.033 +08:00] [INFO] [case.go:302] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:07:31.214 +08:00] [INFO] [case.go:537] ["upstream create table executed"] [tblName=finishmark18000] [2024/04/26 19:07:31.214 +08:00] [INFO] [case.go:569] ["downstream start wait for table"] [tblName=finishmark18000] [2024/04/26 19:07:32.146 +08:00] [ERROR] [case.go:266] ["bank test verify failed"] [error="verify balance failed, accounts0 expect 100000, but got 100151"] [errorVerbose="verify balance failed, accounts0 expect 100000, but got 100151\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1594"] [stack="main.(*bankTest).verify.func1.1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:266\nmain.(*bankTest).verify.func1\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:290\ngithub.com/pingcap/tiflow/pkg/retry.run\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:57\ngithub.com/pingcap/tiflow/pkg/retry.Do\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/pkg/retry/retry_with_opt.go:34\nmain.(*bankTest).verify\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:262\nmain.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:594\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] [2024/04/26 19:07:32.146 +08:00] [PANIC] [case.go:595] ["downstream verify failed"] [error="[CDC:ErrReachMaxTry]reach maximum try: 10, error: verify balance failed, accounts0 expect 100000, but got 100151: verify balance failed, accounts0 expect 100000, but got 100151"] [stack="main.run.func2\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:595\ngolang.org/x/sync/errgroup.(*Group).Go.func1\n\t/go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75"] panic: downstream verify failed goroutine 152 [running]: go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc0005efa80?}) /go/pkg/mod/go.uber.org/zap@v1.23.0/zapcore/entry.go:198 +0x65 go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0004c80d0, {0xc0005ff440, 0x1, 0x1}) /go/pkg/mod/go.uber.org/zap@v1.23.0/zapcore/entry.go:264 +0x3ec go.uber.org/zap.(*Logger).Panic(0xc0005ff440?, {0x8fd2c3?, 0xc000032168?}, {0xc0005ff440, 0x1, 0x1}) /go/pkg/mod/go.uber.org/zap@v1.23.0/logger.go:251 +0x59 github.com/pingcap/log.Panic({0x8fd2c3?, 0x13?}, {0xc0005ff440?, 0x9f2ee8?, 0xc00042f9e0?}) /go/pkg/mod/github.com/pingcap/log@v1.1.1-0.20221116035753-734d527bc87c/global.go:54 +0x8b main.run.func2() /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/bank/case.go:595 +0xa85 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:75 +0x64 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/sync@v0.5.0/errgroup/errgroup.go:72 +0xa5 exit status 2 + stop_tidb_cluster Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858bcc00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42786, start at 2024-04-26 19:06:52.593858292 +0800 CST m=+4.510404089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6858c0a40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:42889, start at 2024-04-26 19:06:52.84808282 +0800 CST m=+4.674987855 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:08:52.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:06:52.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:56:52.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed test leveldb sorter + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.46952.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + set +x + tso='449348804195385345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348804195385345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Fri Apr 26 19:07:38 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937340006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44400, start at 2024-04-26 19:07:00.441146051 +0800 CST m=+4.507603596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685937400006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:44497, start at 2024-04-26 19:07:00.442326159 +0800 CST m=+4.446195842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:00.450 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:00.432 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:00.432 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:07:43 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.4866948671.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:07:46 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d39bbe-1285-40b6-b8a0-ae5aefc1beee {"id":"e3d39bbe-1285-40b6-b8a0-ae5aefc1beee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16d1e18e e3d39bbe-1285-40b6-b8a0-ae5aefc1beee /tidb/cdc/default/default/upstream/7362130627936874277 {"id":7362130627936874277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d39bbe-1285-40b6-b8a0-ae5aefc1beee {"id":"e3d39bbe-1285-40b6-b8a0-ae5aefc1beee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16d1e18e e3d39bbe-1285-40b6-b8a0-ae5aefc1beee /tidb/cdc/default/default/upstream/7362130627936874277 {"id":7362130627936874277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e3d39bbe-1285-40b6-b8a0-ae5aefc1beee {"id":"e3d39bbe-1285-40b6-b8a0-ae5aefc1beee","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16d1e18e e3d39bbe-1285-40b6-b8a0-ae5aefc1beee /tidb/cdc/default/default/upstream/7362130627936874277 {"id":7362130627936874277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/04/26 19:07:46.752 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-normal Info: {"upstream_id":7362130627936874277,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-15962?flush-interval=5s\u0026protocol=csv","create_time":"2024-04-26T19:07:46.821160151+08:00","start_ts":449348806535806977,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348806535806977,"checkpoint_ts":449348806535806977,"checkpoint_time":"2024-04-26 19:07:43.604"} [2024/04/26 19:07:46.904 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7362130627936874277,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-6652?flush-interval=5s\u0026protocol=csv","create_time":"2024-04-26T19:07:46.969210248+08:00","start_ts":449348806535806977,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348806535806977,"checkpoint_ts":449348806535806977,"checkpoint_time":"2024-04-26 19:07:43.604"} [2024/04/26 19:07:47.054 +08:00] [WARN] [replica_config.go:278] ["Attempting to replicate with old value enabled, but the specified protocol must disable old value. CDC will disable old value and continue."] [protocol=csv] Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7362130627936874277,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-29397?flush-interval=5s\u0026protocol=csv","create_time":"2024-04-26T19:07:47.120126783+08:00","start_ts":449348806535806977,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348806535806977,"checkpoint_ts":449348806535806977,"checkpoint_time":"2024-04-26 19:07:43.604"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-15962?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-15962?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-6652?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-6652?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-29397?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Fri Apr 26 19:07:47 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-29397?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a864c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:31041, start at 2024-04-26 19:07:21.907781121 +0800 CST m=+4.464500185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a8fe80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37820, start at 2024-04-26 19:07:22.519412729 +0800 CST m=+4.611188916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later + ls /tmp/tidb_cdc_test/ api_v2 autorandom availability bank cov.api_v2.2455024552.out cov.autorandom.3515835160.out cov.availability.4035540357.out cov.availability.4053240534.out cov.availability.4071540717.out cov.availability.4086240864.out cov.availability.4100241004.out cov.availability.4148141483.out cov.availability.4174741749.out cov.availability.4188441886.out cov.availability.4204242044.out cov.availability.4259142593.out cov.availability.4274742749.out cov.availability.4337743379.out cov.availability.4357743579.out cov.availability.4372143723.out cov.availability.4393143933.out cov.availability.4414944151.out cov.availability.cli.40248.out cov.availability.cli.40420.out cov.bank.cli.49462.out sql_res.api_v2.txt sql_res.autorandom.txt sql_res.availability.txt sql_res.bank.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G0.tar.gz /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/tikv_down.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/tikv2.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/api_v2/tikv_down.log /tmp/tidb_cdc_test/api_v2/tikv2.log /tmp/tidb_cdc_test/api_v2/stdout.log /tmp/tidb_cdc_test/api_v2/tidb.log /tmp/tidb_cdc_test/api_v2/tikv1.log /tmp/tidb_cdc_test/api_v2/tidb-slow.log /tmp/tidb_cdc_test/api_v2/pd1.log /tmp/tidb_cdc_test/api_v2/down_pd.log /tmp/tidb_cdc_test/api_v2/cdc.log /tmp/tidb_cdc_test/api_v2/tikv3.log /tmp/tidb_cdc_test/api_v2/tidb_down.log /tmp/tidb_cdc_test/api_v2/tidb_other.log /tmp/tidb_cdc_test/autorandom/tikv_down.log /tmp/tidb_cdc_test/autorandom/tikv2.log /tmp/tidb_cdc_test/autorandom/stdout.log /tmp/tidb_cdc_test/autorandom/tidb.log /tmp/tidb_cdc_test/autorandom/tikv1.log /tmp/tidb_cdc_test/autorandom/tidb-slow.log /tmp/tidb_cdc_test/autorandom/sync_diff_inspector.log /tmp/tidb_cdc_test/autorandom/pd1.log /tmp/tidb_cdc_test/autorandom/down_pd.log /tmp/tidb_cdc_test/autorandom/cdc.log /tmp/tidb_cdc_test/autorandom/tikv3.log /tmp/tidb_cdc_test/autorandom/tidb_down.log /tmp/tidb_cdc_test/autorandom/tidb_other.log /tmp/tidb_cdc_test/bank/tikv2/db/000025.log /tmp/tidb_cdc_test/bank/tikv2/db/000018.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000006.log /tmp/tidb_cdc_test/bank/tikv_down.log /tmp/tidb_cdc_test/bank/tikv2.log /tmp/tidb_cdc_test/bank/stdout.log /tmp/tidb_cdc_test/bank/tidb.log /tmp/tidb_cdc_test/bank/tikv1.log /tmp/tidb_cdc_test/bank/tikv_down/db/000028.log /tmp/tidb_cdc_test/bank/tidb-slow.log /tmp/tidb_cdc_test/bank/tiflash/log/error.log /tmp/tidb_cdc_test/bank/tiflash/log/server.log /tmp/tidb_cdc_test/bank/tiflash/log/proxy.log /tmp/tidb_cdc_test/bank/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/bank/down_pd/hot-region/000004.log /tmp/tidb_cdc_test/bank/down_pd/region-meta/000005.log /tmp/tidb_cdc_test/bank/pd1.log /tmp/tidb_cdc_test/bank/down_pd.log /tmp/tidb_cdc_test/bank/cdc.log /tmp/tidb_cdc_test/bank/tikv3.log /tmp/tidb_cdc_test/bank/tidb_down.log /tmp/tidb_cdc_test/bank/tidb_other.log /tmp/tidb_cdc_test/bank/tikv3/db/000029.log /tmp/tidb_cdc_test/bank/tikv1-2024-04-26T19-07-03.171.log /tmp/tidb_cdc_test/bank/pd1/hot-region/000004.log /tmp/tidb_cdc_test/bank/pd1/region-meta/000005.log /tmp/tidb_cdc_test/bank/tikv1/db/000029.log /tmp/tidb_cdc_test/bank/tikv1/db/000033.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/tikv_down.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/tikv2.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/api_v2/tikv_down.log /tmp/tidb_cdc_test/api_v2/tikv2.log /tmp/tidb_cdc_test/api_v2/stdout.log /tmp/tidb_cdc_test/api_v2/tidb.log /tmp/tidb_cdc_test/api_v2/tikv1.log /tmp/tidb_cdc_test/api_v2/tidb-slow.log /tmp/tidb_cdc_test/api_v2/pd1.log /tmp/tidb_cdc_test/api_v2/down_pd.log /tmp/tidb_cdc_test/api_v2/cdc.log /tmp/tidb_cdc_test/api_v2/tikv3.log /tmp/tidb_cdc_test/api_v2/tidb_down.log /tmp/tidb_cdc_test/api_v2/tidb_other.log /tmp/tidb_cdc_test/autorandom/tikv_down.log /tmp/tidb_cdc_test/autorandom/tikv2.log /tmp/tidb_cdc_test/autorandom/stdout.log /tmp/tidb_cdc_test/autorandom/tidb.log /tmp/tidb_cdc_test/autorandom/tikv1.log /tmp/tidb_cdc_test/autorandom/tidb-slow.log /tmp/tidb_cdc_test/autorandom/sync_diff_inspector.log /tmp/tidb_cdc_test/autorandom/pd1.log /tmp/tidb_cdc_test/autorandom/down_pd.log /tmp/tidb_cdc_test/autorandom/cdc.log /tmp/tidb_cdc_test/autorandom/tikv3.log /tmp/tidb_cdc_test/autorandom/tidb_down.log /tmp/tidb_cdc_test/autorandom/tidb_other.log /tmp/tidb_cdc_test/bank/tikv2/db/000025.log /tmp/tidb_cdc_test/bank/tikv2/db/000018.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0003/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0005/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0007/000006.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000004.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000005.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000007.log /tmp/tidb_cdc_test/bank/cdc_data/tmp/sorter/0001/000006.log /tmp/tidb_cdc_test/bank/tikv_down.log table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later /tmp/tidb_cdc_test/bank/tikv2.log /tmp/tidb_cdc_test/bank/stdout.log /tmp/tidb_cdc_test/bank/tidb.log /tmp/tidb_cdc_test/bank/tikv1.log table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later /tmp/tidb_cdc_test/bank/tikv_down/db/000028.log /tmp/tidb_cdc_test/bank/tidb-slow.log /tmp/tidb_cdc_test/bank/tiflash/log/error.log /tmp/tidb_cdc_test/bank/tiflash/log/server.log /tmp/tidb_cdc_test/bank/tiflash/log/proxy.log /tmp/tidb_cdc_test/bank/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/bank/down_pd/hot-region/000004.log /tmp/tidb_cdc_test/bank/down_pd/region-meta/000005.log /tmp/tidb_cdc_test/bank/pd1.log /tmp/tidb_cdc_test/bank/down_pd.log /tmp/tidb_cdc_test/bank/cdc.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later /tmp/tidb_cdc_test/bank/tikv3.log /tmp/tidb_cdc_test/bank/tidb_down.log /tmp/tidb_cdc_test/bank/tidb_other.log /tmp/tidb_cdc_test/bank/tikv3/db/000029.log /tmp/tidb_cdc_test/bank/tikv1-2024-04-26T19-07-03.171.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a87f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:30960, start at 2024-04-26 19:07:21.985006623 +0800 CST m=+4.612818758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.981 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.981 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a864c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:31041, start at 2024-04-26 19:07:21.907781121 +0800 CST m=+4.464500185 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:21.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:21.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:21.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.35005.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348812455018497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348812455018497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later /tmp/tidb_cdc_test/bank/pd1/hot-region/000004.log /tmp/tidb_cdc_test/bank/pd1/region-meta/000005.log /tmp/tidb_cdc_test/bank/tikv1/db/000029.log /tmp/tidb_cdc_test/bank/tikv1/db/000033.log + ls -alh log-G0.tar.gz -rw-r--r-- 1 jenkins jenkins 63M Apr 26 19:08 log-G0.tar.gz [Pipeline] archiveArtifacts Archiving artifacts start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Recording fingerprints [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G0' table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later table test.finish_mark not exists for 5-th check, retry later table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later table test.finish_mark exists check table exists success check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a9164000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37757, start at 2024-04-26 19:07:22.595078149 +0800 CST m=+4.789394687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685a8fe80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:37820, start at 2024-04-26 19:07:22.519412729 +0800 CST m=+4.611188916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:09:22.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:07:22.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:57:22.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:08:17 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.4163741639.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:08:18 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:08:20 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/baf50035-c1a0-4587-98bf-07b21e21d068 {"id":"baf50035-c1a0-4587-98bf-07b21e21d068","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a174cc890 baf50035-c1a0-4587-98bf-07b21e21d068 /tidb/cdc/default/default/upstream/7362130713200988071 {"id":7362130713200988071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/baf50035-c1a0-4587-98bf-07b21e21d068 {"id":"baf50035-c1a0-4587-98bf-07b21e21d068","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a174cc890 baf50035-c1a0-4587-98bf-07b21e21d068 /tidb/cdc/default/default/upstream/7362130713200988071 {"id":7362130713200988071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/baf50035-c1a0-4587-98bf-07b21e21d068 {"id":"baf50035-c1a0-4587-98bf-07b21e21d068","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a174cc890 baf50035-c1a0-4587-98bf-07b21e21d068 /tidb/cdc/default/default/upstream/7362130713200988071 {"id":7362130713200988071,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: fe7d1164-846b-4b33-a921-5f727f5848a8 Info: {"upstream_id":7362130713200988071,"namespace":"default","id":"fe7d1164-846b-4b33-a921-5f727f5848a8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-04-26T19:08:20.739740437+08:00","start_ts":449348815401779201,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348815401779201,"checkpoint_ts":449348815401779201,"checkpoint_time":"2024-04-26 19:08:17.425"} table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed can't found update row with old value run task failed 1-th time, retry later try a VALID cdc server command [Fri Apr 26 19:08:23 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.4148441486.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed can't found update row with old value run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:08:26 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45bf59ea-e78e-4e32-be39-109089ab3ccf {"id":"45bf59ea-e78e-4e32-be39-109089ab3ccf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a168c5c03 45bf59ea-e78e-4e32-be39-109089ab3ccf /tidb/cdc/default/default/upstream/7362130554494719349 {"id":7362130554494719349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45bf59ea-e78e-4e32-be39-109089ab3ccf {"id":"45bf59ea-e78e-4e32-be39-109089ab3ccf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a168c5c03 45bf59ea-e78e-4e32-be39-109089ab3ccf /tidb/cdc/default/default/upstream/7362130554494719349 {"id":7362130554494719349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45bf59ea-e78e-4e32-be39-109089ab3ccf {"id":"45bf59ea-e78e-4e32-be39-109089ab3ccf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a168c5c03 45bf59ea-e78e-4e32-be39-109089ab3ccf /tidb/cdc/default/default/upstream/7362130554494719349 {"id":7362130554494719349,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check_old_value_enabled /tmp/tidb_cdc_test/multi_changefeed [2024/04/26 19:08:24.703 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816723247114,\"commit-ts\":449348816723247115,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/04/26 19:08:24.703 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816723247126,\"commit-ts\":449348816723247127,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/04/26 19:08:24.704 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816736354310,\"commit-ts\":449348816736354311,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] [2024/04/26 19:08:24.764 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816723247114,\"commit-ts\":449348816723247115,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}],\"pre-columns\":null}"] [2024/04/26 19:08:24.764 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816723247126,\"commit-ts\":449348816723247127,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}],\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":2,\"default\":null}]}"] [2024/04/26 19:08:24.764 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"start-ts\":449348816736354310,\"commit-ts\":449348816736354311,\"row-id\":1,\"table\":{\"db-name\":\"multi_changefeed\",\"tbl-name\":\"t1\",\"tbl-id\":82,\"is-partition\":false},\"column-infos\":[{\"ID\":1,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}},{\"ID\":2,\"IsPKHandle\":false,\"VirtualGenCol\":false,\"Ft\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null}}],\"columns\":null,\"pre-columns\":[{\"name\":\"id\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":27,\"value\":1,\"default\":null},{\"name\":\"val\",\"type\":3,\"charset\":\"binary\",\"collation\":\"binary\",\"flag\":65,\"value\":3,\"default\":null}]}"] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:08:29 CST 2024] <<<<<< run test case multi_changefeed success! >>>>>> table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 2024-04-26 19:08:32.633317 W | etcdserver: could not get cluster response from http://127.0.0.1:2380: Get "http://127.0.0.1:2380/members": dial tcp 127.0.0.1:2380: connect: connection refused Verifying upstream PD is started... table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readproportion"="0" "updateproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "mysql.db"="sorter" "readallfields"="true" "dotransactions"="false" "threadcount"="10" "insertproportion"="0" "recordcount"="3000" "requestdistribution"="uniform" "workload"="core" "mysql.port"="4000" "mysql.user"="root" ********************************************** Run finished, takes 1.010008229s INSERT - Takes(s): 1.0, Count: 3000, OPS: 3021.2, Avg(us): 3283, Min(us): 1166, Max(us): 28195, 95th(us): 10000, 99th(us): 18000 [Fri Apr 26 19:08:34 CST 2024] <<<<<< START cdc server in sorter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.4708447086.out server --log-file /tmp/tidb_cdc_test/sorter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sorter/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:08:37 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7dce8e33-9e94-4b84-b92b-bc61e5310721 {"id":"7dce8e33-9e94-4b84-b92b-bc61e5310721","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16b2251a 7dce8e33-9e94-4b84-b92b-bc61e5310721 /tidb/cdc/default/default/upstream/7362130587629920559 {"id":7362130587629920559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7dce8e33-9e94-4b84-b92b-bc61e5310721 {"id":"7dce8e33-9e94-4b84-b92b-bc61e5310721","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16b2251a 7dce8e33-9e94-4b84-b92b-bc61e5310721 /tidb/cdc/default/default/upstream/7362130587629920559 {"id":7362130587629920559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7dce8e33-9e94-4b84-b92b-bc61e5310721 {"id":"7dce8e33-9e94-4b84-b92b-bc61e5310721","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a16b2251a 7dce8e33-9e94-4b84-b92b-bc61e5310721 /tidb/cdc/default/default/upstream/7362130587629920559 {"id":7362130587629920559,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sorter.cli.47126.out cli changefeed create -c ticdc-leveldb-sorter-test-20599 --start-ts=449348804195385345 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' --sort-engine=unified Create changefeed successfully! ID: ticdc-leveldb-sorter-test-20599 Info: {"upstream_id":7362130587629920559,"namespace":"default","id":"ticdc-leveldb-sorter-test-20599","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:08:37.668769944+08:00","start_ts":449348804195385345,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348804195385345,"checkpoint_ts":449348804195385345,"checkpoint_time":"2024-04-26 19:07:34.676"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table sorter.usertable not exists for 1-th check, retry later table sorter.usertable exists table sorter.check1 not exists for 1-th check, retry later table sorter.check1 not exists for 2-th check, retry later table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later table sorter.check1 exists check diff successfully table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f8f700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53164, start at 2024-04-26 19:08:44.419944246 +0800 CST m=+5.990313065 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log check diff successfully valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri Apr 26 19:08:46 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.4157441576.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None table sorter.check2 not exists for 1-th check, retry later table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table sorter.check2 exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "operationcount"="0" "scanproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.port"="4000" "dotransactions"="false" "mysql.db"="sorter" "workload"="core" "recordcount"="3000" "mysql.host"="127.0.0.1" "readproportion"="0" "updateproportion"="0" "threadcount"="10" "insertproportion"="0" ********************************************** ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 33192 0 --:--:-- --:--:-- --:--:-- 34428 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:06:28.739","puller_resolved_ts":"2024-04-26 19:06:28.789","last_synced_ts":"2024-04-26 19:06:21.888","now_ts":"2024-04-26 19:08:39.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:06:28.739","puller_resolved_ts":"2024-04-26' '19:06:28.789","last_synced_ts":"2024-04-26' '19:06:21.888","now_ts":"2024-04-26' '19:08:39.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:06:28.739","puller_resolved_ts":"2024-04-26' '19:06:28.789","last_synced_ts":"2024-04-26' '19:06:21.888","now_ts":"2024-04-26' '19:08:39.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later Run finished, takes 2.074517506s INSERT - Takes(s): 2.1, Count: 3000, OPS: 1449.8, Avg(us): 6802, Min(us): 879, Max(us): 83574, 95th(us): 61000, 99th(us): 75000 table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later table sorter.check3 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 6-th check, retry later table sorter.check3 exists check diff successfully Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sorter.usertable2 exists table sorter.check4 not exists for 1-th check, retry later table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sorter.check4 not exists for 2-th check, retry later table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sorter.check4 not exists for 3-th check, retry later table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32731, start at 2024-04-26 19:09:00.396169427 +0800 CST m=+4.344306426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log table sorter.check4 exists check diff successfully table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting CDC state checker... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:09:05 CST 2024] <<<<<< run test case sorter success! >>>>>> # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 4-th check, retry later start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri Apr 26 19:09:06 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 5-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 6-th check, retry later table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f8f700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53164, start at 2024-04-26 19:08:44.419944246 +0800 CST m=+5.990313065 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 7-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table processor_delay.t50 exists table multi_tables_ddl_test.finish_mark not exists for 8-th check, retry later check diff failed 1-th time, retry later [Fri Apr 26 19:09:04 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.3512535127.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:09:07 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3efe1d1-0d22-4907-9a91-383f79b99e2b {"id":"b3efe1d1-0d22-4907-9a91-383f79b99e2b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a17278b18 b3efe1d1-0d22-4907-9a91-383f79b99e2b /tidb/cdc/default/default/upstream/7362130719389733114 {"id":7362130719389733114,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3efe1d1-0d22-4907-9a91-383f79b99e2b {"id":"b3efe1d1-0d22-4907-9a91-383f79b99e2b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a17278b18 b3efe1d1-0d22-4907-9a91-383f79b99e2b /tidb/cdc/default/default/upstream/7362130719389733114 {"id":7362130719389733114,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3efe1d1-0d22-4907-9a91-383f79b99e2b {"id":"b3efe1d1-0d22-4907-9a91-383f79b99e2b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a17278b18 b3efe1d1-0d22-4907-9a91-383f79b99e2b /tidb/cdc/default/default/upstream/7362130719389733114 {"id":7362130719389733114,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.35182.out cli changefeed create --start-ts=449348812455018497 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a1f6a092-1491-4fc4-bedb-a8ee474c0091 Info: {"upstream_id":7362130719389733114,"namespace":"default","id":"a1f6a092-1491-4fc4-bedb-a8ee474c0091","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:09:07.993355576+08:00","start_ts":449348812455018497,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348812455018497,"checkpoint_ts":449348812455018497,"checkpoint_time":"2024-04-26 19:08:06.184"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table multi_tables_ddl_test.finish_mark not exists for 9-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:09:16 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 10-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348831334629380, "checkpoint_time": "2024-04-26 19:09:18.204", "error": null }' + echo '{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348831334629380, "checkpoint_time": "2024-04-26 19:09:18.204", "error": null }' { "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348831334629380, "checkpoint_time": "2024-04-26 19:09:18.204", "error": null } ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348831334629380, '"checkpoint_time":' '"2024-04-26' '19:09:18.204",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348831334629380, '"checkpoint_time":' '"2024-04-26' '19:09:18.204",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348831387058179, "checkpoint_time": "2024-04-26 19:09:18.404", "error": null }' + echo '{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348831387058179, "checkpoint_time": "2024-04-26 19:09:18.404", "error": null }' { "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348831387058179, "checkpoint_time": "2024-04-26 19:09:18.404", "error": null } ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348831387058179, '"checkpoint_time":' '"2024-04-26' '19:09:18.404",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348831387058179, '"checkpoint_time":' '"2024-04-26' '19:09:18.404",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348822054731801, "checkpoint_time": "2024-04-26 19:08:42.804", "error": { "time": "2024-04-26T19:08:45.515471229+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348822565650441]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348822054731801, "checkpoint_time": "2024-04-26 19:08:42.804", "error": { "time": "2024-04-26T19:08:45.515471229+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348822565650441]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7362130627936874277, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348822054731801, "checkpoint_time": "2024-04-26 19:08:42.804", "error": { "time": "2024-04-26T19:08:45.515471229+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348822565650441]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348822054731801, '"checkpoint_time":' '"2024-04-26' '19:08:42.804",' '"error":' '{' '"time":' '"2024-04-26T19:08:45.515471229+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348822565650441]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362130627936874277, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348822054731801, '"checkpoint_time":' '"2024-04-26' '19:08:42.804",' '"error":' '{' '"time":' '"2024-04-26T19:08:45.515471229+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348822565650441]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''106'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348822565650441]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''106'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:118, Type:rename table, State:done, SchemaState:public, SchemaID:80, TableID:106, RowCount:0, ArgLen:0, start time: 2024-04-26 19:08:44.753 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, query: rename table t11 to t9, startTs: 449348822565650441. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348822565650441]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '106', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully table force_replicate_table.t3 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:09:20 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> table force_replicate_table.t4 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32731, start at 2024-04-26 19:09:00.396169427 +0800 CST m=+4.344306426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/processor_panic/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table force_replicate_table.t6 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449348832259735556" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/04/26 19:09:24.242 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:09:24.242 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:09:24.243 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348832259735556] [resolvedTs=449348832259735556] [2024/04/26 19:09:24.243 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=310.89Β΅s] [2024/04/26 19:09:24.243 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=316.276Β΅s] [2024/04/26 19:09:24.243 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:09:24.243 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:09:24.249 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:09:24.253 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:09:24.254 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:09:24.255 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:09:24.254 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:09:24.259 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:09:24.261 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 19:09:24.261 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=449348832259735556] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 19:09:24.261 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Fri Apr 26 19:09:24 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f93600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53104, start at 2024-04-26 19:08:44.64188003 +0800 CST m=+6.307962551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.632 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.632 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c685f8f700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--l6gn5, pid:53164, start at 2024-04-26 19:08:44.419944246 +0800 CST m=+5.990313065 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:10:44.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:08:44.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:58:44.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57179.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348834071937025 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348834071937025 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d58001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42522, start at 2024-04-26 19:09:29.351419956 +0800 CST m=+4.230583985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686280900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48390, start at 2024-04-26 19:09:32.585437373 +0800 CST m=+7.877062195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.580 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.580 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9CFF2F10D557A < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:09:35 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 Verifying downstream PD is started... Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:09:39 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862cdf80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:47001, start at 2024-04-26 19:09:37.543479606 +0800 CST m=+4.258393136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_panic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_panic/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32668, start at 2024-04-26 19:09:00.408672748 +0800 CST m=+4.436192014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686089500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:32731, start at 2024-04-26 19:09:00.396169427 +0800 CST m=+4.344306426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:00.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:00.372 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:00.372 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.36714.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348838070419457 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348838070419457 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449348838070419457 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:09:44 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3675136753.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:09:47 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/756f3e81-ba52-47f2-9b8f-0350ffee0740 {"id":"756f3e81-ba52-47f2-9b8f-0350ffee0740","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a18a37d8e 756f3e81-ba52-47f2-9b8f-0350ffee0740 /tidb/cdc/default/default/upstream/7362131141291879789 {"id":7362131141291879789,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/756f3e81-ba52-47f2-9b8f-0350ffee0740 {"id":"756f3e81-ba52-47f2-9b8f-0350ffee0740","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a18a37d8e 756f3e81-ba52-47f2-9b8f-0350ffee0740 /tidb/cdc/default/default/upstream/7362131141291879789 {"id":7362131141291879789,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/756f3e81-ba52-47f2-9b8f-0350ffee0740 {"id":"756f3e81-ba52-47f2-9b8f-0350ffee0740","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a18a37d8e 756f3e81-ba52-47f2-9b8f-0350ffee0740 /tidb/cdc/default/default/upstream/7362131141291879789 {"id":7362131141291879789,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449348838070419457 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.36811.out cli changefeed create --start-ts=449348838070419457 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362131141291879789,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:09:47.611663689+08:00","start_ts":449348838070419457,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348838070419457,"checkpoint_ts":449348838070419457,"checkpoint_time":"2024-04-26 19:09:43.899"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686352480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50398, start at 2024-04-26 19:09:46.011108552 +0800 CST m=+5.122155759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.018 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.002 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.002 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/error.log Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting CDC state checker... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686381e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42117, start at 2024-04-26 19:09:49.054670187 +0800 CST m=+4.499952358 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:49.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:49.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:49.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d58001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42522, start at 2024-04-26 19:09:29.351419956 +0800 CST m=+4.230583985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862cdf80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:47001, start at 2024-04-26 19:09:37.543479606 +0800 CST m=+4.258393136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686280900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48390, start at 2024-04-26 19:09:32.585437373 +0800 CST m=+7.877062195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.580 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.580 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686352480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50398, start at 2024-04-26 19:09:46.011108552 +0800 CST m=+5.122155759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.018 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.002 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.002 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864bba00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45087, start at 2024-04-26 19:10:09.147980657 +0800 CST m=+6.296632163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log start tidb cluster in /tmp/tidb_cdc_test/processor_panic Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting CDC state checker... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d5c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42427, start at 2024-04-26 19:09:29.306438403 +0800 CST m=+4.272150229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68624d58001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:42522, start at 2024-04-26 19:09:29.351419956 +0800 CST m=+4.230583985 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:29.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:29.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:29.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686381e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42117, start at 2024-04-26 19:09:49.054670187 +0800 CST m=+4.499952358 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:49.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:49.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:49.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862ce0c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:46916, start at 2024-04-26 19:09:37.542167609 +0800 CST m=+4.328479411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862cdf80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b6c8m, pid:47001, start at 2024-04-26 19:09:37.543479606 +0800 CST m=+4.258393136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:37.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:37.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:37.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.51207.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348847860711425 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348847860711425 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:10:21 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ true == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + true + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.5124451246.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:24 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:10:24 CST 2024] <<<<<< START cdc server in processor_panic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.512495130851310.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Fri Apr 26 19:10:25 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.5725757259.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:27 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63fcdef4-0bb7-4683-a408-a81db46df932 {"id":"63fcdef4-0bb7-4683-a408-a81db46df932","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352db2 63fcdef4-0bb7-4683-a408-a81db46df932 /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63fcdef4-0bb7-4683-a408-a81db46df932 {"id":"63fcdef4-0bb7-4683-a408-a81db46df932","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352db2 63fcdef4-0bb7-4683-a408-a81db46df932 /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63fcdef4-0bb7-4683-a408-a81db46df932 {"id":"63fcdef4-0bb7-4683-a408-a81db46df932","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/e34de24a-01ba-4629-b75c-f76aa4e02a3c {"id":"e34de24a-01ba-4629-b75c-f76aa4e02a3c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352d8f e34de24a-01ba-4629-b75c-f76aa4e02a3c /tidb/cdc/default/__cdc_meta__/owner/22318f1a19352db2 63fcdef4-0bb7-4683-a408-a81db46df932 /tidb/cdc/default/default/upstream/7362131302682836949 {"id":7362131302682836949,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.cli.51370.out cli changefeed create --start-ts=449348847860711425 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: a9d7ebb7-2527-4176-bf99-100cf6f746fc Info: {"upstream_id":7362131302682836949,"namespace":"default","id":"a9d7ebb7-2527-4176-bf99-100cf6f746fc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:10:28.065422081+08:00","start_ts":449348847860711425,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348847860711425,"checkpoint_ts":449348847860711425,"checkpoint_time":"2024-04-26 19:10:21.246"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6862810c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48358, start at 2024-04-26 19:09:32.640184132 +0800 CST m=+8.088506861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686280900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--b502w, pid:48390, start at 2024-04-26 19:09:32.585437373 +0800 CST m=+7.877062195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:32.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:32.580 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:32.580 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686354a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50299, start at 2024-04-26 19:09:46.160176452 +0800 CST m=+5.332997873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686352480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--rk2d5, pid:50398, start at 2024-04-26 19:09:46.011108552 +0800 CST m=+5.122155759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:46.018 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:46.002 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:46.002 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:10:28 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5450654508.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:28 GMT < Content-Length: 656 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/008039ad-804e-4ee0-89a8-2188ab3ab09a {"id":"008039ad-804e-4ee0-89a8-2188ab3ab09a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a186cf387 008039ad-804e-4ee0-89a8-2188ab3ab09a /tidb/cdc/default/default/upstream/7362131068006281649 {"id":7362131068006281649,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/008039ad-804e-4ee0-89a8-2188ab3ab09a {"id":"008039ad-804e-4ee0-89a8-2188ab3ab09a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a186cf387 008039ad-804e-4ee0-89a8-2188ab3ab09a /tidb/cdc/default/default/upstream/7362131068006281649 {"id":7362131068006281649,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/008039ad-804e-4ee0-89a8-2188ab3ab09a {"id":"008039ad-804e-4ee0-89a8-2188ab3ab09a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a186cf387 008039ad-804e-4ee0-89a8-2188ab3ab09a /tidb/cdc/default/default/upstream/7362131068006281649 {"id":7362131068006281649,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57299.out cli changefeed create --start-ts=449348834071937025 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.52064.out cli tso query --pd=http://127.0.0.1:2379 [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362131068006281649,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:10:28.967233799+08:00","start_ts":449348834071937025,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348834071937025,"checkpoint_ts":449348834071937025,"checkpoint_time":"2024-04-26 19:09:28.646"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table test.simple not exists for 1-th check, retry later + set +x + tso='449348849894424577 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348849894424577 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:31 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:10:31 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5457954581.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348848896180230, "checkpoint_time": "2024-04-26 19:10:25.196", "error": null }' + echo '{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348848896180230, "checkpoint_time": "2024-04-26 19:10:25.196", "error": null }' { "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348848896180230, "checkpoint_time": "2024-04-26 19:10:25.196", "error": null } ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348848896180230, '"checkpoint_time":' '"2024-04-26' '19:10:25.196",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348848896180230, '"checkpoint_time":' '"2024-04-26' '19:10:25.196",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:34 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:10:34 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.5464054642.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57557.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:37 GMT < Content-Length: 1141 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fa37f7f6-2df6-4e8f-9031-534e0d3182ee {"id":"fa37f7f6-2df6-4e8f-9031-534e0d3182ee","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544c4 fa37f7f6-2df6-4e8f-9031-534e0d3182ee /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fa37f7f6-2df6-4e8f-9031-534e0d3182ee {"id":"fa37f7f6-2df6-4e8f-9031-534e0d3182ee","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544c4 fa37f7f6-2df6-4e8f-9031-534e0d3182ee /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cff7f62-07ad-4c5e-aea2-7a1eb003a421 {"id":"6cff7f62-07ad-4c5e-aea2-7a1eb003a421","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/8ad558d7-40bf-48e2-b299-43ae101236ab {"id":"8ad558d7-40bf-48e2-b299-43ae101236ab","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/fa37f7f6-2df6-4e8f-9031-534e0d3182ee {"id":"fa37f7f6-2df6-4e8f-9031-534e0d3182ee","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1955448e 8ad558d7-40bf-48e2-b299-43ae101236ab /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544b1 6cff7f62-07ad-4c5e-aea2-7a1eb003a421 /tidb/cdc/default/__cdc_meta__/owner/22318f1a195544c4 fa37f7f6-2df6-4e8f-9031-534e0d3182ee /tidb/cdc/default/default/upstream/7362131325394849786 {"id":7362131325394849786,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.cli.54709.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 92f4b117-5ff4-417d-8451-02985ef8d582 Info: {"upstream_id":7362131325394849786,"namespace":"default","id":"92f4b117-5ff4-417d-8451-02985ef8d582","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:10:38.178032108+08:00","start_ts":449348852281507845,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"csv","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348852281507845,"checkpoint_ts":449348852281507845,"checkpoint_time":"2024-04-26 19:10:38.110"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864bba00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45087, start at 2024-04-26 19:10:09.147980657 +0800 CST m=+6.296632163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348851478298631, "checkpoint_time": "2024-04-26 19:10:35.046", "error": null }' + echo '{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348851478298631, "checkpoint_time": "2024-04-26 19:10:35.046", "error": null }' { "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348851478298631, "checkpoint_time": "2024-04-26 19:10:35.046", "error": null } ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449348851478298631, '"checkpoint_time":' '"2024-04-26' '19:10:35.046",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449348851478298631, '"checkpoint_time":' '"2024-04-26' '19:10:35.046",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57620.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7362131068006281649,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:10:28.967233799+08:00","start_ts":449348834071937025,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348851504513027,"checkpoint_ts":449348851478298631,"checkpoint_time":"2024-04-26 19:10:35.046"} PASS coverage: 3.0% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57644.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68637fc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42022, start at 2024-04-26 19:09:48.953401697 +0800 CST m=+4.457541609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:48.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:48.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:48.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686381e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:42117, start at 2024-04-26 19:09:49.054670187 +0800 CST m=+4.499952358 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:11:49.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:09:49.050 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-18:59:49.050 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:10:42 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4603746039.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348853850701825, "checkpoint_time": "2024-04-26 19:10:44.096", "error": null }' + echo '{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348853850701825, "checkpoint_time": "2024-04-26 19:10:44.096", "error": null }' { "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348853850701825, "checkpoint_time": "2024-04-26 19:10:44.096", "error": null } ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348853850701825, '"checkpoint_time":' '"2024-04-26' '19:10:44.096",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348853850701825, '"checkpoint_time":' '"2024-04-26' '19:10:44.096",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57695.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449348853850701825 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:10:46 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e17e759d-3a0c-4f49-8e99-d67113fbafd0 {"id":"e17e759d-3a0c-4f49-8e99-d67113fbafd0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19830f91 e17e759d-3a0c-4f49-8e99-d67113fbafd0 /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e17e759d-3a0c-4f49-8e99-d67113fbafd0 {"id":"e17e759d-3a0c-4f49-8e99-d67113fbafd0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19830f91 e17e759d-3a0c-4f49-8e99-d67113fbafd0 /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e17e759d-3a0c-4f49-8e99-d67113fbafd0 {"id":"e17e759d-3a0c-4f49-8e99-d67113fbafd0","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19830f91 e17e759d-3a0c-4f49-8e99-d67113fbafd0 /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57734.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 36580 1.0 0.0 2000972 125376 ? Sl 19:09 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714129781522838201.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 36584 0.9 0.0 1680624 132928 ? Sl 19:09 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714129781526210807.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 36663 1.0 0.0 1598248 142200 ? Sl 19:09 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714129781586534357.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2507 0 --:--:-- --:--:-- --:--:-- 2531 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:10:48.750","puller_resolved_ts":"2024-04-26 19:10:47.799","last_synced_ts":"2024-04-26 19:10:41.899","now_ts":"2024-04-26 19:10:49.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:10:48.750","puller_resolved_ts":"2024-04-26' '19:10:47.799","last_synced_ts":"2024-04-26' '19:10:41.899","now_ts":"2024-04-26' '19:10:49.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:10:48.750","puller_resolved_ts":"2024-04-26' '19:10:47.799","last_synced_ts":"2024-04-26' '19:10:41.899","now_ts":"2024-04-26' '19:10:49.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362131068006281649,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:10:49.569083112+08:00","start_ts":449348855240327172,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348855240327172,"checkpoint_ts":449348855240327172,"checkpoint_time":"2024-04-26 19:10:49.397"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348855240327172, "checkpoint_time": "2024-04-26 19:10:49.397", "error": null }' + echo '{ "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348855240327172, "checkpoint_time": "2024-04-26 19:10:49.397", "error": null }' { "upstream_id": 7362131068006281649, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348855240327172, "checkpoint_time": "2024-04-26 19:10:49.397", "error": null } ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348855240327172, '"checkpoint_time":' '"2024-04-26' '19:10:49.397",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131068006281649, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348855240327172, '"checkpoint_time":' '"2024-04-26' '19:10:49.397",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57815.out cli unsafe delete-service-gc-safepoint Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57835.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864b9940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45052, start at 2024-04-26 19:10:09.034147956 +0800 CST m=+6.298706221 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.050 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.046 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.046 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6864bba00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:45087, start at 2024-04-26 19:10:09.147980657 +0800 CST m=+6.296632163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:12:09.170 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:10:09.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:00:09.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.48620.out cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57885.out cli unsafe resolve-lock --region=50 PASS + set +x + tso='449348857676955649 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348857676955649 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:10:59 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.4863748639.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.57899.out cli unsafe resolve-lock --region=50 --ts=449348856879251459 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 41724 1180 --:--:-- --:--:-- --:--:-- 53000 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:11:02 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd44663e-08b2-4f9c-8a49-0b9b5275b29c {"id":"bd44663e-08b2-4f9c-8a49-0b9b5275b29c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19c8048a bd44663e-08b2-4f9c-8a49-0b9b5275b29c /tidb/cdc/default/default/upstream/7362131430174318069 {"id":7362131430174318069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd44663e-08b2-4f9c-8a49-0b9b5275b29c {"id":"bd44663e-08b2-4f9c-8a49-0b9b5275b29c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19c8048a bd44663e-08b2-4f9c-8a49-0b9b5275b29c /tidb/cdc/default/default/upstream/7362131430174318069 {"id":7362131430174318069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd44663e-08b2-4f9c-8a49-0b9b5275b29c {"id":"bd44663e-08b2-4f9c-8a49-0b9b5275b29c","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19c8048a bd44663e-08b2-4f9c-8a49-0b9b5275b29c /tidb/cdc/default/default/upstream/7362131430174318069 {"id":7362131430174318069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.48673.out cli changefeed create --start-ts=449348857676955649 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: fdac5d4a-cf40-4db2-9cd4-3ce37f50bd20 Info: {"upstream_id":7362131430174318069,"namespace":"default","id":"fdac5d4a-cf40-4db2-9cd4-3ce37f50bd20","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:11:02.740544951+08:00","start_ts":449348857676955649,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348857676955649,"checkpoint_ts":449348857676955649,"checkpoint_time":"2024-04-26 19:10:58.692"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 175 100 175 0 0 133k 0 --:--:-- --:--:-- --:--:-- 170k { "version": "v6.5.9-1-gc3f88235b", "git_hash": "c3f88235bc30ad0173bd585b87a683ca1cf9280f", "id": "f1ee37bd-4424-4692-b19d-37d680eaa4dc", "pid": 57262, "is_owner": true }cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:04 CST 2024] <<<<<< run test case cli success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/04/26 19:11:19.412 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_8] [numQueries=200] [2024/04/26 19:11:19.416 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_5] [numQueries=200] [2024/04/26 19:11:19.417 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_2] [numQueries=200] [2024/04/26 19:11:19.418 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_9] [numQueries=200] [2024/04/26 19:11:19.419 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_4] [numQueries=200] [2024/04/26 19:11:19.420 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_0] [numQueries=200] [2024/04/26 19:11:19.421 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_7] [numQueries=200] [2024/04/26 19:11:19.422 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_1] [numQueries=200] [2024/04/26 19:11:19.471 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_3] [numQueries=200] [2024/04/26 19:11:19.479 +08:00] [INFO] [main.go:91] ["Start inserting data"] [tableName=test_table_6] [numQueries=200] [2024/04/26 19:11:19.672 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_1] [2024/04/26 19:11:19.672 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_5] [2024/04/26 19:11:19.672 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_4] [2024/04/26 19:11:19.675 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_8] [2024/04/26 19:11:19.676 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_2] [2024/04/26 19:11:19.678 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_0] [2024/04/26 19:11:19.678 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_9] [2024/04/26 19:11:19.678 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_7] [2024/04/26 19:11:19.717 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_3] [2024/04/26 19:11:19.723 +08:00] [INFO] [main.go:109] ["Finished inserting data"] [tableName=test_table_6] table test.end_mark_table not exists for 1-th check, retry later table test.end_mark_table not exists for 2-th check, retry later table test.end_mark_table not exists for 3-th check, retry later ***************** properties ***************** "insertproportion"="0" "workload"="core" "readallfields"="true" "mysql.db"="changefeed_auto_stop_1" "operationcount"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="20" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "threadcount"="4" "updateproportion"="0" ********************************************** Run finished, takes 8.931453ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4313.6, Avg(us): 1682, Min(us): 917, Max(us): 4232, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="4" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "mysql.port"="4000" "recordcount"="20" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_2" "workload"="core" "readproportion"="0" "operationcount"="0" "mysql.user"="root" ********************************************** Run finished, takes 12.714768ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2644.4, Avg(us): 2407, Min(us): 1496, Max(us): 5141, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "scanproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "mysql.user"="root" "workload"="core" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" "mysql.db"="changefeed_auto_stop_3" "recordcount"="20" "threadcount"="4" "readproportion"="0" "updateproportion"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.316838ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3356.3, Avg(us): 1865, Min(us): 1025, Max(us): 4466, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "mysql.db"="changefeed_auto_stop_4" "threadcount"="4" "readproportion"="0" "readallfields"="true" "workload"="core" "operationcount"="0" "updateproportion"="0" "mysql.user"="root" "mysql.port"="4000" "scanproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="20" ********************************************** Run finished, takes 10.461031ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3719.1, Avg(us): 1995, Min(us): 1015, Max(us): 5040, 95th(us): 6000, 99th(us): 6000 [Fri Apr 26 19:11:20 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.4680446806.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:11:23 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:11:23 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.4686946871.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:11:26 GMT < Content-Length: 865 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138d596-f836-46b0-8882-4c3a6d385bce {"id":"4138d596-f836-46b0-8882-4c3a6d385bce","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a7c 4138d596-f836-46b0-8882-4c3a6d385bce /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138d596-f836-46b0-8882-4c3a6d385bce {"id":"4138d596-f836-46b0-8882-4c3a6d385bce","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a7c 4138d596-f836-46b0-8882-4c3a6d385bce /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4138d596-f836-46b0-8882-4c3a6d385bce {"id":"4138d596-f836-46b0-8882-4c3a6d385bce","address":"127.0.0.1:8302","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/capture/f1a9476c-caa2-478c-bdcf-a05e83cc203a {"id":"f1a9476c-caa2-478c-bdcf-a05e83cc203a","address":"127.0.0.1:8301","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a5b f1a9476c-caa2-478c-bdcf-a05e83cc203a /tidb/cdc/default/__cdc_meta__/owner/22318f1a192c3a7c 4138d596-f836-46b0-8882-4c3a6d385bce /tidb/cdc/default/default/upstream/7362131266281382566 {"id":7362131266281382566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 71f36dd7-ba1d-422f-892c-dca9177cf69b normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=71f36dd7-ba1d-422f-892c-dca9177cf69b + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 71f36dd7-ba1d-422f-892c-dca9177cf69b -s table test.end_mark_table not exists for 4-th check, retry later + info='{ "upstream_id": 7362131266281382566, "namespace": "default", "id": "71f36dd7-ba1d-422f-892c-dca9177cf69b", "state": "normal", "checkpoint_tso": 449348847076638723, "checkpoint_time": "2024-04-26 19:10:18.255", "error": null }' + echo '{ "upstream_id": 7362131266281382566, "namespace": "default", "id": "71f36dd7-ba1d-422f-892c-dca9177cf69b", "state": "normal", "checkpoint_tso": 449348847076638723, "checkpoint_time": "2024-04-26 19:10:18.255", "error": null }' { "upstream_id": 7362131266281382566, "namespace": "default", "id": "71f36dd7-ba1d-422f-892c-dca9177cf69b", "state": "normal", "checkpoint_tso": 449348847076638723, "checkpoint_time": "2024-04-26 19:10:18.255", "error": null } ++ echo '{' '"upstream_id":' 7362131266281382566, '"namespace":' '"default",' '"id":' '"71f36dd7-ba1d-422f-892c-dca9177cf69b",' '"state":' '"normal",' '"checkpoint_tso":' 449348847076638723, '"checkpoint_time":' '"2024-04-26' '19:10:18.255",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131266281382566, '"namespace":' '"default",' '"id":' '"71f36dd7-ba1d-422f-892c-dca9177cf69b",' '"state":' '"normal",' '"checkpoint_tso":' 449348847076638723, '"checkpoint_time":' '"2024-04-26' '19:10:18.255",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later table test.end_mark_table not exists for 5-th check, retry later table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later table test.end_mark_table not exists for 6-th check, retry later table changefeed_auto_stop_2.usertable not exists for 2-th check, retry later *************************** 1. row *************************** count(distinct region_id): 6 table region_merge.t1 exists check diff failed 1-th time, retry later check diff successfully table test.end_mark_table not exists for 7-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:32 CST 2024] <<<<<< run test case region_merge success! >>>>>> table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later + workdir=/tmp/tidb_cdc_test/csv_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_partition_table ++ date + echo '[Fri Apr 26 19:11:28 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>>' [Fri Apr 26 19:11:28 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml table test.end_mark_table not exists for 8-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later table partition_table.t not exists for 1-th check, retry later table test.end_mark_table not exists for 9-th check, retry later table changefeed_auto_stop_4.usertable exists check diff successfully [Fri Apr 26 19:11:28 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.5214752149.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:11:31 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44caf658-913c-48c6-93c8-0f0039a8eb23 {"id":"44caf658-913c-48c6-93c8-0f0039a8eb23","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1953de25 44caf658-913c-48c6-93c8-0f0039a8eb23 /tidb/cdc/default/default/upstream/7362131266041450615 {"id":7362131266041450615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44caf658-913c-48c6-93c8-0f0039a8eb23 {"id":"44caf658-913c-48c6-93c8-0f0039a8eb23","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1953de25 44caf658-913c-48c6-93c8-0f0039a8eb23 /tidb/cdc/default/default/upstream/7362131266041450615 {"id":7362131266041450615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44caf658-913c-48c6-93c8-0f0039a8eb23 {"id":"44caf658-913c-48c6-93c8-0f0039a8eb23","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1953de25 44caf658-913c-48c6-93c8-0f0039a8eb23 /tidb/cdc/default/default/upstream/7362131266041450615 {"id":7362131266041450615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.52186.out cli changefeed create --start-ts=449348849894424577 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: c06861b3-a12c-4736-be30-2fbf76af8474 Info: {"upstream_id":7362131266041450615,"namespace":"default","id":"c06861b3-a12c-4736-be30-2fbf76af8474","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:11:31.649883985+08:00","start_ts":449348849894424577,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348849894424577,"checkpoint_ts":449348849894424577,"checkpoint_time":"2024-04-26 19:10:29.004"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table split_region.test1 not exists for 1-th check, retry later table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table split_region.test2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:37 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> table partition_table.t not exists for 2-th check, retry later table test.end_mark_table not exists for 10-th check, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later table test.end_mark_table exists check diff successfully + '[' 0 -eq 143 ']' + sleep 1 cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:41 CST 2024] <<<<<< run test case processor_panic success! >>>>>> + true + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedPreEmit=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_panic.5124451246.out server --log-file /tmp/tidb_cdc_test/processor_panic/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_panic/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= table partition_table.finish_mark not exists for 2-th check, retry later table consistent_replicate_gbk.check1 not exists for 2-th check, retry later table partition_table.finish_mark not exists for 3-th check, retry later The 1 times to try to start tidb cluster... table consistent_replicate_gbk.check1 exists check diff successfully table partition_table.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:46 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sinkv2/eventsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.4629746299.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:11:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {ClusterID:default ID:default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 Info:{"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} Status:0xc00121b698 TaskPositions:map[da9dda72-605a-466d-a08c-0425c1bcfcd9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348868291166218, checkpointTs: 449348868291166218, state: Replicating tableID: 82, resolvedTs: 449348870650724359, checkpointTs: 449348868291166218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da9dda72-605a-466d-a08c-0425c1bcfcd9 {"id":"da9dda72-605a-466d-a08c-0425c1bcfcd9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19831141 da9dda72-605a-466d-a08c-0425c1bcfcd9 /tidb/cdc/default/default/changefeed/info/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} /tidb/cdc/default/default/changefeed/status/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":449348868291166218,"min-table-barrier-ts":449348870912868358,"admin-job-type":0} /tidb/cdc/default/default/task/position/da9dda72-605a-466d-a08c-0425c1bcfcd9/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {ClusterID:default ID:default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 Info:{"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} Status:0xc00121b698 TaskPositions:map[da9dda72-605a-466d-a08c-0425c1bcfcd9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348868291166218, checkpointTs: 449348868291166218, state: Replicating tableID: 82, resolvedTs: 449348870650724359, checkpointTs: 449348868291166218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da9dda72-605a-466d-a08c-0425c1bcfcd9 {"id":"da9dda72-605a-466d-a08c-0425c1bcfcd9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19831141 da9dda72-605a-466d-a08c-0425c1bcfcd9 /tidb/cdc/default/default/changefeed/info/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compr+ grep -q 'failed to get info:' ession":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} /tidb/cdc/default/default/changefeed/status/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":449348868291166218,"min-table-barrier-ts":449348870912868358,"admin-job-type":0} /tidb/cdc/default/default/task/position/da9dda72-605a-466d-a08c-0425c1bcfcd9/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {ClusterID:default ID:default/4b95d3df-0b91-4994-b70e-c9ef8ea62356 Info:{"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http%3A%2F%2F127.0.0.1%3A24927%2F","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} Status:0xc00121b698 TaskPositions:map[da9dda72-605a-466d-a08c-0425c1bcfcd9:{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}] pendingPatches:[] skipPatchesInThisTick:false} tableID: 84, resolvedTs: 449348868291166218, checkpointTs: 449348868291166218, state: Replicating tableID: 82, resolvedTs: 449348870650724359, checkpointTs: 449348868291166218, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/da9dda72-605a-466d-a08c-0425c1bcfcd9 {"id":"da9dda72-605a-466d-a08c-0425c1bcfcd9","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a19831141 da9dda72-605a-466d-a08c-0425c1bcfcd9 /tidb/cdc/default/default/changefeed/info/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"upstream-id":7362131354402248046,"namespace":"default","changefeed-id":"4b95d3df-0b91-4994-b70e-c9ef8ea62356","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-04-26T19:10:46.148313078+08:00","start-ts":449348854371581953,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348854384689154} /tidb/cdc/default/default/changefeed/status/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":449348868291166218,"min-table-barrier-ts":449348870912868358,"admin-job-type":0} /tidb/cdc/default/default/task/position/da9dda72-605a-466d-a08c-0425c1bcfcd9/4b95d3df-0b91-4994-b70e-c9ef8ea62356 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362131354402248046 {"id":7362131354402248046,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } check diff successfully [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table partition_table.finish_mark not exists for 6-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:11:51 CST 2024] <<<<<< run test case split_region success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table partition_table.finish_mark not exists for 8-th check, retry later check_redo_resolved_ts 4b95d3df-0b91-4994-b70e-c9ef8ea62356 449348872419934213 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/4b95d3df-0b91-4994-b70e-c9ef8ea62356/meta global resolved ts 449348871069892616 not forward to 449348872419934213 run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 4b95d3df-0b91-4994-b70e-c9ef8ea62356 449348872419934213 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/4b95d3df-0b91-4994-b70e-c9ef8ea62356/meta run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [2024/04/26 19:11:58.490 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:11:58.491 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:11:58.494 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:11:58.494 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348871069892616] [resolvedTs=449348872760983553] [2024/04/26 19:11:58.494 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:11:58.494 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:11:58.494 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:11:58.495 +08:00] [INFO] [s3.go:399] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/04/26 19:11:58.498 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=4.421896ms] [2024/04/26 19:11:58.509 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:11:58.510 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=16.289248ms] [2024/04/26 19:11:58.512 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:11:58.513 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.513 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.513 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:11:58.514 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:11:58.514 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:11:58.514 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:11:58.518 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:11:58.519 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 19:11:58.519 +08:00] [WARN] [redo.go:280] ["apply DDL"] [ddl="{\"StartTs\":449348871069892611,\"CommitTs\":449348871069892616,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [2024/04/26 19:11:58.520 +08:00] [INFO] [mysql_ddl_sink.go:160] ["Start exec DDL"] [DDL="{\"StartTs\":449348871069892611,\"CommitTs\":449348871069892616,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":86,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\"}"] [namespace=] [changefeed=] [2024/04/26 19:11:58.521 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=449348871069892611] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050: Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] table partition_table.finish_mark not exists for 10-th check, retry later [2024/04/26 19:11:58.721 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=449348872760983553] [2024/04/26 19:11:58.721 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [tableID=86] [checkpointTs=449348872760983553] [2024/04/26 19:11:58.822 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449348872760983553] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 19:11:58.822 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] Apply redo log successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully [Fri Apr 26 19:11:59 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT table partition_table.finish_mark not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b73800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33499, start at 2024-04-26 19:11:59.24880491 +0800 CST m=+4.584618183 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b8aa80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48149, start at 2024-04-26 19:12:00.719382719 +0800 CST m=+4.276349477 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log [Pipeline] // container table partition_table.finish_mark not exists for 12-th check, retry later [Pipeline] } [Pipeline] // withEnv Starting CDC state checker... [Pipeline] } # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table partition_table.finish_mark not exists for 13-th check, retry later Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... table partition_table.finish_mark not exists for 14-th check, retry later table partition_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:12:09 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table foreign_key.finish_mark not exists for 8-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table foreign_key.finish_mark not exists for 9-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table foreign_key.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... table foreign_key.finish_mark not exists for 11-th check, retry later Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:12:11 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... Verifying downstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b73800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33499, start at 2024-04-26 19:11:59.24880491 +0800 CST m=+4.584618183 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table foreign_key.finish_mark exists [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... [Pipeline] // node check diff successfully [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:12:23 CST 2024] <<<<<< run test case foreign_key success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd4400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47555, start at 2024-04-26 19:12:21.777896072 +0800 CST m=+4.315226908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b8aa80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48149, start at 2024-04-26 19:12:00.719382719 +0800 CST m=+4.276349477 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d7e140007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36412, start at 2024-04-26 19:12:32.653154581 +0800 CST m=+4.302621580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b72040015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33412, start at 2024-04-26 19:11:59.130425211 +0800 CST m=+4.538003916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.105 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.105 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b73800019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:33499, start at 2024-04-26 19:11:59.24880491 +0800 CST m=+4.584618183 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:59.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:59.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:59.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.37330.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + set +x + tso='449348884472791041 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348884472791041 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:12:41 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.3736837370.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:12:44 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e826ba2-2e26-423e-b66f-249eb4e07604 {"id":"5e826ba2-2e26-423e-b66f-249eb4e07604","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b5aa48f 5e826ba2-2e26-423e-b66f-249eb4e07604 /tidb/cdc/default/default/upstream/7362131912513388078 {"id":7362131912513388078,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e826ba2-2e26-423e-b66f-249eb4e07604 {"id":"5e826ba2-2e26-423e-b66f-249eb4e07604","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b5aa48f 5e826ba2-2e26-423e-b66f-249eb4e07604 /tidb/cdc/default/default/upstream/7362131912513388078 {"id":7362131912513388078,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e826ba2-2e26-423e-b66f-249eb4e07604 {"id":"5e826ba2-2e26-423e-b66f-249eb4e07604","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b5aa48f 5e826ba2-2e26-423e-b66f-249eb4e07604 /tidb/cdc/default/default/upstream/7362131912513388078 {"id":7362131912513388078,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.37422.out cli changefeed create --start-ts=449348884472791041 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic Create changefeed successfully! ID: 584adbd2-dc80-439c-9a29-3fb38b8aa4f9 Info: {"upstream_id":7362131912513388078,"namespace":"default","id":"584adbd2-dc80-439c-9a29-3fb38b8aa4f9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-04-26T19:12:44.645461448+08:00","start_ts":449348884472791041,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348884472791041,"checkpoint_ts":449348884472791041,"checkpoint_time":"2024-04-26 19:12:40.910"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x go: downloading github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.14.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/spf13/cobra v1.6.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.24.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading k8s.io/apimachinery v0.24.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.60.1 go: downloading github.com/google/gofuzz v1.1.0 go: downloading sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.1 go: downloading k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 go: downloading github.com/go-logr/logr v1.2.0 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd4400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47555, start at 2024-04-26 19:12:21.777896072 +0800 CST m=+4.315226908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadd40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49675, start at 2024-04-26 19:12:52.097419053 +0800 CST m=+6.173603836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b87940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48058, start at 2024-04-26 19:12:00.502541009 +0800 CST m=+4.127682947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686b8aa80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:48149, start at 2024-04-26 19:12:00.719382719 +0800 CST m=+4.276349477 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:00.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:00.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:00.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.52186.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348888038211585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348888038211585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting CDC state checker... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d7e140007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36412, start at 2024-04-26 19:12:32.653154581 +0800 CST m=+4.302621580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2512 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:12:57.850","puller_resolved_ts":"2024-04-26 19:12:57.949","last_synced_ts":"2024-04-26 19:10:41.899","now_ts":"2024-04-26 19:12:59.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:12:57.850","puller_resolved_ts":"2024-04-26' '19:12:57.949","last_synced_ts":"2024-04-26' '19:10:41.899","now_ts":"2024-04-26' '19:12:59.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:12:57.850","puller_resolved_ts":"2024-04-26' '19:12:57.949","last_synced_ts":"2024-04-26' '19:10:41.899","now_ts":"2024-04-26' '19:12:59.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd5000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47469, start at 2024-04-26 19:12:21.84549696 +0800 CST m=+4.431629454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.851 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686cd4400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--s5gf9, pid:47555, start at 2024-04-26 19:12:21.777896072 +0800 CST m=+4.315226908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:21.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:21.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:21.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:13:13 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.5157451576.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:13:16 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e9c55d-bff5-40ce-9f24-e0edd93f9090 {"id":"87e9c55d-bff5-40ce-9f24-e0edd93f9090","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bd5928e 87e9c55d-bff5-40ce-9f24-e0edd93f9090 /tidb/cdc/default/default/upstream/7362132010804890080 {"id":7362132010804890080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e9c55d-bff5-40ce-9f24-e0edd93f9090 {"id":"87e9c55d-bff5-40ce-9f24-e0edd93f9090","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bd5928e 87e9c55d-bff5-40ce-9f24-e0edd93f9090 /tidb/cdc/default/default/upstream/7362132010804890080 {"id":7362132010804890080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87e9c55d-bff5-40ce-9f24-e0edd93f9090 {"id":"87e9c55d-bff5-40ce-9f24-e0edd93f9090","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bd5928e 87e9c55d-bff5-40ce-9f24-e0edd93f9090 /tidb/cdc/default/default/upstream/7362132010804890080 {"id":7362132010804890080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d80700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36319, start at 2024-04-26 19:12:32.819599593 +0800 CST m=+4.529112541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686d7e140007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:36412, start at 2024-04-26 19:12:32.653154581 +0800 CST m=+4.302621580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:32.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:32.645 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:32.645 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed skip some DDLs in tidb v4.0.x [Fri Apr 26 19:13:18 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.4030540307.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:13:21 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/370c21a6-82af-4f09-b226-9239b39b763b {"id":"370c21a6-82af-4f09-b226-9239b39b763b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bed508e 370c21a6-82af-4f09-b226-9239b39b763b /tidb/cdc/default/default/upstream/7362132055676829072 {"id":7362132055676829072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/370c21a6-82af-4f09-b226-9239b39b763b {"id":"370c21a6-82af-4f09-b226-9239b39b763b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bed508e 370c21a6-82af-4f09-b226-9239b39b763b /tidb/cdc/default/default/upstream/7362132055676829072 {"id":7362132055676829072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/370c21a6-82af-4f09-b226-9239b39b763b {"id":"370c21a6-82af-4f09-b226-9239b39b763b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1bed508e 370c21a6-82af-4f09-b226-9239b39b763b /tidb/cdc/default/default/upstream/7362132055676829072 {"id":7362132055676829072,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706e580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37982, start at 2024-04-26 19:13:20.795443932 +0800 CST m=+4.615445989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.790 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.790 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadd40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49675, start at 2024-04-26 19:12:52.097419053 +0800 CST m=+6.173603836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:13:38.855 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/04/26 19:13:38.857 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:13:38.857 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7362131912513388078] [2024/04/26 19:13:38.857 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/04/26 19:13:38.857 +08:00] [INFO] [client.go:408] ["[pd] create pd client with endpoints"] [pd-address="[127.0.0.1:2379]"] [2024/04/26 19:13:38.858 +08:00] [INFO] [base_client.go:384] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:13:38.858 +08:00] [INFO] [base_client.go:106] ["[pd] init cluster id"] [cluster-id=7362131912513388078] [2024/04/26 19:13:38.858 +08:00] [INFO] [client.go:701] ["[pd] tso dispatcher created"] [dc-location=global] [2024/04/26 19:13:38.859 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/04/26 19:13:38.860 +08:00] [INFO] [region_cache.go:2485] ["change store resolve state"] [store=2] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706e580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37982, start at 2024-04-26 19:13:20.795443932 +0800 CST m=+4.615445989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.790 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.790 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadcc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49639, start at 2024-04-26 19:12:52.112978171 +0800 CST m=+6.339863254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c686eadd40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--kshm6, pid:49675, start at 2024-04-26 19:12:52.097419053 +0800 CST m=+6.173603836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:14:52.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:12:52.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:02:52.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed [Fri Apr 26 19:13:49 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.5321253214.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:13:48.877 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/04/26 19:13:48.877 +08:00] [INFO] [base_client.go:143] ["[pd] exit member loop due to context canceled"] [2024/04/26 19:13:48.877 +08:00] [INFO] [client.go:783] ["[pd] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/04/26 19:13:48.877 +08:00] [INFO] [client.go:719] ["[pd] exit tso dispatcher"] [dc-location=global] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:13:52 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/268a77dc-3e5b-4b42-a393-fbde4346f1c1 {"id":"268a77dc-3e5b-4b42-a393-fbde4346f1c1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c644190 268a77dc-3e5b-4b42-a393-fbde4346f1c1 /tidb/cdc/default/default/upstream/7362132130794692169 {"id":7362132130794692169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/268a77dc-3e5b-4b42-a393-fbde4346f1c1 {"id":"268a77dc-3e5b-4b42-a393-fbde4346f1c1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c644190 268a77dc-3e5b-4b42-a393-fbde4346f1c1 /tidb/cdc/default/default/upstream/7362132130794692169 {"id":7362132130794692169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/268a77dc-3e5b-4b42-a393-fbde4346f1c1 {"id":"268a77dc-3e5b-4b42-a393-fbde4346f1c1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c644190 268a77dc-3e5b-4b42-a393-fbde4346f1c1 /tidb/cdc/default/default/upstream/7362132130794692169 {"id":7362132130794692169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... table test.t2 not exists for 1-th check, retry later table test.t2 not exists for 2-th check, retry later Verifying downstream PD is started... table test.t2 not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.t2 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 5-th check, retry later ***************** properties ***************** "recordcount"="20" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" "readallfields"="true" "mysql.port"="4000" "scanproportion"="0" "threadcount"="4" "workload"="core" "readproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "mysql.db"="changefeed_error" ********************************************** Run finished, takes 10.488033ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3443.5, Avg(us): 1842, Min(us): 941, Max(us): 4945, 95th(us): 5000, 99th(us): 5000 [Fri Apr 26 19:13:51 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5230452306.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:13:54 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf {"id":"9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1819 9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf {"id":"9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1819 9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf {"id":"9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1819 9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.52371.out cli changefeed create --start-ts=449348888038211585 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7362131918513645715,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:13:55.153833376+08:00","start_ts":449348888038211585,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585,"checkpoint_time":"2024-04-26 19:12:54.511"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348888038211585, "checkpoint_time": "2024-04-26 19:12:54.511", "error": { "time": "2024-04-26T19:13:55.219898398+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at 449348888038211585" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348888038211585, "checkpoint_time": "2024-04-26 19:12:54.511", "error": { "time": "2024-04-26T19:13:55.219898398+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at 449348888038211585" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348888038211585, "checkpoint_time": "2024-04-26 19:12:54.511", "error": { "time": "2024-04-26T19:13:55.219898398+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at 449348888038211585" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449348888038211585, '"checkpoint_time":' '"2024-04-26' '19:12:54.511",' '"error":' '{' '"time":' '"2024-04-26T19:13:55.219898398+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at '449348888038211585"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449348888038211585, '"checkpoint_time":' '"2024-04-26' '19:12:54.511",' '"error":' '{' '"time":' '"2024-04-26T19:13:55.219898398+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at '449348888038211585"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at 449348888038211585' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348888038211285 is earlier than or equal to GC safepoint at 449348888038211585 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.52472.out cli changefeed resume -c changefeed-error PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x table changefeed_error.usertable not exists for 1-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff successfully ***************** properties ***************** "mysql.db"="changefeed_error" "updateproportion"="0" "mysql.port"="4000" "insertproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "workload"="core" "scanproportion"="0" "threadcount"="4" "requestdistribution"="uniform" "dotransactions"="false" "operationcount"="0" "readproportion"="0" "recordcount"="20" "mysql.user"="root" ********************************************** Run finished, takes 3.82491ms INSERT - Takes(s): 0.0, Count: 20, OPS: 7371.5, Avg(us): 671, Min(us): 386, Max(us): 1468, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + [[ /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + [[ /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706b940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37927, start at 2024-04-26 19:13:20.655554828 +0800 CST m=+4.545727832 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68706e580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:37982, start at 2024-04-26 19:13:20.795443932 +0800 CST m=+4.615445989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:15:20.804 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:13:20.790 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:03:20.790 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.41945.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348906145808385 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348906145808385 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449348906145808385 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:14:03 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4198941991.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.t2 not exists for 6-th check, retry later table test.t2 not exists for 7-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + [[ /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/9dceadb4-beaf-4a9b-9d99-1c34e5bd8ecf + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:06 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7216af7-f4e9-47c1-a241-f96f00cd6235 {"id":"b7216af7-f4e9-47c1-a241-f96f00cd6235","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c99b390 b7216af7-f4e9-47c1-a241-f96f00cd6235 /tidb/cdc/default/default/upstream/7362132263593281981 {"id":7362132263593281981,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7216af7-f4e9-47c1-a241-f96f00cd6235 {"id":"b7216af7-f4e9-47c1-a241-f96f00cd6235","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c99b390 b7216af7-f4e9-47c1-a241-f96f00cd6235 /tidb/cdc/default/default/upstream/7362132263593281981 {"id":7362132263593281981,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7216af7-f4e9-47c1-a241-f96f00cd6235 {"id":"b7216af7-f4e9-47c1-a241-f96f00cd6235","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1c99b390 b7216af7-f4e9-47c1-a241-f96f00cd6235 /tidb/cdc/default/default/upstream/7362132263593281981 {"id":7362132263593281981,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449348906145808385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.42057.out cli changefeed create --start-ts=449348906145808385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362132263593281981,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:14:07.326827403+08:00","start_ts":449348906145808385,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348906145808385,"checkpoint_ts":449348906145808385,"checkpoint_time":"2024-04-26 19:14:03.586"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 table test.t2 not exists for 8-th check, retry later table test.t2 not exists for 9-th check, retry later table test.t2 exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:14:12 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> ***************** properties ***************** "fieldcount"="3" "operationcount"="0" "insertproportion"="0" "mysql.db"="consistent_replicate_nfs" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "updateproportion"="0" "workload"="core" "mysql.port"="4000" "mysql.user"="root" "readproportion"="0" "recordcount"="50000" "threadcount"="10" "readallfields"="true" "scanproportion"="0" "dotransactions"="false" ********************************************** check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:14:12 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5273952741.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91dbaccd-83ee-4936-abaa-1d96c7834fff {"id":"91dbaccd-83ee-4936-abaa-1d96c7834fff","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b188a 91dbaccd-83ee-4936-abaa-1d96c7834fff /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362131918513645715,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:13:55.153833376+08:00","start-ts":449348888038211585,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:14:13.176263777+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348908655837188} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348904894070788,"min-table-barrier-ts":449348904894070788,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91dbaccd-83ee-4936-abaa-1d96c7834fff {"id":"91dbaccd-83ee-4936-abaa-1d96c7834fff","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b188a 91dbaccd-83ee-4936-abaa-1d96c7834fff /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362131918513645715,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:13:55.153833376+08:00","start-ts":449348888038211585,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:14:13.176263777+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348908655837188} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348904894070788,"min-table-barrier-ts":449348904894070788,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91dbaccd-83ee-4936-abaa-1d96c7834fff {"id":"91dbaccd-83ee-4936-abaa-1d96c7834fff","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b188a 91dbaccd-83ee-4936-abaa-1d96c7834fff /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362131918513645715,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-04-26T19:13:55.153833376+08:00","start-ts":449348888038211585,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:14:13.176263777+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v6.5.9-1-gc3f88235b","epoch":449348908655837188} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348904894070788,"min-table-barrier-ts":449348904894070788,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348904894070788, "checkpoint_time": "2024-04-26 19:13:58.811", "error": { "time": "2024-04-26T19:14:13.176263777+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348904894070788, "checkpoint_time": "2024-04-26 19:13:58.811", "error": { "time": "2024-04-26T19:14:13.176263777+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348904894070788, "checkpoint_time": "2024-04-26 19:13:58.811", "error": { "time": "2024-04-26T19:14:13.176263777+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348904894070788, '"checkpoint_time":' '"2024-04-26' '19:13:58.811",' '"error":' '{' '"time":' '"2024-04-26T19:14:13.176263777+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348904894070788, '"checkpoint_time":' '"2024-04-26' '19:13:58.811",' '"error":' '{' '"time":' '"2024-04-26T19:14:13.176263777+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.52861.out cli changefeed remove -c changefeed-error Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449348904894070788 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:14:17 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5297052972.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f changefeed is working normally rts: 449348909362577400->449348909899972600 checkpoint: 449348909624721400->449348910162116600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/04/26 19:14:17.086 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=87.148563ms] [namespace=] [changefeed=] run task successfully resume changefeed e7a39791-d843-454c-a9ca-4415d1a4427f from 449348909585137666 check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:20 GMT < Content-Length: 680 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cf32a9b-691d-4fed-a60b-ee5ad3adb95b {"id":"9cf32a9b-691d-4fed-a60b-ee5ad3adb95b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b18ad 9cf32a9b-691d-4fed-a60b-ee5ad3adb95b /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cf32a9b-691d-4fed-a60b-ee5ad3adb95b {"id":"9cf32a9b-691d-4fed-a60b-ee5ad3adb95b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b18ad 9cf32a9b-691d-4fed-a60b-ee5ad3adb95b /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cf32a9b-691d-4fed-a60b-ee5ad3adb95b {"id":"9cf32a9b-691d-4fed-a60b-ee5ad3adb95b","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b18ad 9cf32a9b-691d-4fed-a60b-ee5ad3adb95b /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53026.out cli changefeed create --start-ts=449348888038211585 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7362131918513645715,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:14:20.922672818+08:00","start_ts":449348888038211585,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585,"checkpoint_time":"2024-04-26 19:12:54.511"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 918 0 --:--:-- --:--:-- --:--:-- 924 + info='{"state":"normal","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585}' + echo '{"state":"normal","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585}' {"state":"normal","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585} ++ echo '{"state":"normal","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later changefeed is working normally rts: 449348910162116600->449348910686404600 checkpoint: 449348909585137660->449348910686404600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/04/26 19:14:20.006 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=449348909572292635] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=] [changefeed=] [error="Error 1007: Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... INSERT - Takes(s): 10.0, Count: 18066, OPS: 1807.7, Avg(us): 5055, Min(us): 751, Max(us): 1346336, 95th(us): 6000, 99th(us): 85000 check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 3902 0 --:--:-- --:--:-- --:--:-- 3953 + info='{"state":"warning","resolved_ts":449348902993526793,"checkpoint_ts":449348902993526793,"last_warning":{"time":"2024-04-26T19:14:23.053940079+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449348902993526793,"checkpoint_ts":449348902993526793,"last_warning":{"time":"2024-04-26T19:14:23.053940079+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449348902993526793,"checkpoint_ts":449348902993526793,"last_warning":{"time":"2024-04-26T19:14:23.053940079+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449348902993526793,"checkpoint_ts":449348902993526793,"last_warning":{"time":"2024-04-26T19:14:23.053940079+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449348902993526793,"checkpoint_ts":449348902993526793,"last_warning":{"time":"2024-04-26T19:14:23.053940079+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53095.out cli changefeed remove -c changefeed-error-1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449348902993526793 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x changefeed is working normally rts: 449348910948548600->449348910974500900 checkpoint: 449348910948548600->449348910974500900 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/04/26 19:14:23.080 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=81.204605ms] [namespace=] [changefeed=] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:14:24 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5315353155.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 resume changefeed e7a39791-d843-454c-a9ca-4415d1a4427f from 449348910974500899 check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:28 GMT < Content-Length: 680 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705711e5-5925-41ac-aa80-341f8035f393 {"id":"705711e5-5925-41ac-aa80-341f8035f393","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1902 705711e5-5925-41ac-aa80-341f8035f393 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705711e5-5925-41ac-aa80-341f8035f393 {"id":"705711e5-5925-41ac-aa80-341f8035f393","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1902 705711e5-5925-41ac-aa80-341f8035f393 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/705711e5-5925-41ac-aa80-341f8035f393 {"id":"705711e5-5925-41ac-aa80-341f8035f393","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1902 705711e5-5925-41ac-aa80-341f8035f393 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53213.out cli changefeed create --start-ts=449348888038211585 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7362131918513645715,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:14:28.3954209+08:00","start_ts":449348888038211585,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348888038211585,"checkpoint_ts":449348888038211585,"checkpoint_time":"2024-04-26 19:12:54.511"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:14:28.40113504+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:14:28.40113504+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:14:28.40113504+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:14:28.40113504+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:14:28.40113504+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348888038211585 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53311.out cli changefeed remove -c changefeed-error-2 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 42938 0 --:--:-- --:--:-- --:--:-- 45187 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:14:03.586","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:14:27.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:14:03.586","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:14:27.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:14:03.586","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:14:27.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:14:30 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.5335753359.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed is working normally rts: 449348912783556600->449348913307844600 checkpoint: 449348912783556600->449348913307844600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/04/26 19:14:27.051 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=449348910974500892] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f INSERT - Takes(s): 20.0, Count: 30780, OPS: 1539.5, Avg(us): 6208, Min(us): 751, Max(us): 1346336, 95th(us): 74000, 99th(us): 86000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:33 GMT < Content-Length: 680 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 {"id":"fc2afa53-7b1e-462f-a146-9e6cd91cc1c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1927 fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 {"id":"fc2afa53-7b1e-462f-a146-9e6cd91cc1c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1927 fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 {"id":"fc2afa53-7b1e-462f-a146-9e6cd91cc1c5","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1b8b1927 fc2afa53-7b1e-462f-a146-9e6cd91cc1c5 /tidb/cdc/default/default/upstream/7362131918513645715 {"id":7362131918513645715,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53422.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_error/conf/enable_redo.toml Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7362131918513645715,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:14:33.648569068+08:00","start_ts":449348914003574785,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/changefeed_error/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348914003574785,"checkpoint_ts":449348914003574785,"checkpoint_time":"2024-04-26 19:14:33.561"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348914003574785, '"checkpoint_time":' '"2024-04-26' '19:14:33.561",' '"error":' '{' '"time":' '"2024-04-26T19:14:33.74059751+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later changefeed is working normally rts: 449348913307844600->449348913307844600 checkpoint: 449348913307844600->449348913543774200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/04/26 19:14:34.177 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=79.097094ms] [namespace=] [changefeed=] run task successfully resume changefeed e7a39791-d843-454c-a9ca-4415d1a4427f from 449348913543774238 check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348914003574785, "checkpoint_time": "2024-04-26 19:14:33.561", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348914003574785, '"checkpoint_time":' '"2024-04-26' '19:14:33.561",' '"error":' '{' '"time":' '"2024-04-26T19:14:33.74059751+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later run task failed 1-th time, retry later + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687500cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40683, start at 2024-04-26 19:14:35.732535368 +0800 CST m=+4.356750204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log Starting CDC state checker... check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' null '}' ++ jq -r .state # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53609.out cli changefeed pause -c changefeed-initialize-error chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": { "time": "2024-04-26T19:14:33.74059751+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' '{' '"time":' '"2024-04-26T19:14:33.74059751+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' '{' '"time":' '"2024-04-26T19:14:33.74059751+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53691.out cli changefeed resume -c changefeed-initialize-error changefeed is working normally rts: 449348915404996600->449348915929284600 checkpoint: 449348915404996600->449348915929284600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/04/26 19:14:35.008 +08:00] [INFO] [mysql_ddl_sink.go:120] ["Execute DDL failed, but error can be ignored"] [startTs=449348913543774234] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=] [changefeed=] [error="Error 1050: Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f PASS coverage: 2.2% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null }' + echo '{ "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null }' { "upstream_id": 7362131918513645715, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348915392937985, "checkpoint_time": "2024-04-26 19:14:38.861", "error": null } ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362131918513645715, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348915392937985, '"checkpoint_time":' '"2024-04-26' '19:14:38.861",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.53790.out cli changefeed remove -c changefeed-initialize-error Verifying downstream PD is started... INSERT - Takes(s): 30.0, Count: 45193, OPS: 1506.7, Avg(us): 6432, Min(us): 726, Max(us): 1346336, 95th(us): 75000, 99th(us): 86000 Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449348915392937985 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:14:43 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release changefeed is working normally rts: 449348916191428600->449348916453572600 checkpoint: 449348916152107000->449348916715716600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/04/26 19:14:42.459 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=308.674106ms] [namespace=] [changefeed=] run task successfully resume changefeed e7a39791-d843-454c-a9ca-4415d1a4427f from 449348916152107021 check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f Run finished, takes 32.890441883s INSERT - Takes(s): 32.9, Count: 50000, OPS: 1520.5, Avg(us): 6342, Min(us): 726, Max(us): 1346336, 95th(us): 74000, 99th(us): 86000 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_nfs.usertable exists cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later check diff failed 2-th time, retry later check_ts_forward e7a39791-d843-454c-a9ca-4415d1a4427f ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7362132130794692169 449348918488334341 449348903297351682 changefeed is working normally rts: 449348918026436600->449348918550724600 checkpoint: 449348918026436600->449348918550724600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/04/26 19:14:47.145 +08:00] [INFO] [mysql_ddl_sink.go:195] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=146.231109ms] [namespace=] [changefeed=] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:14:52 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43094, start at 2024-04-26 19:14:50.366292331 +0800 CST m=+4.385909230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log run task successfully check_changefeed_state http://127.0.0.1:2379 f812b9b0-53bd-4346-9651-fd5739b66faf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=f812b9b0-53bd-4346-9651-fd5739b66faf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f812b9b0-53bd-4346-9651-fd5739b66faf -s + info='{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' + echo '{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' { "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null } ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"stopped",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"stopped",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362132130794692169 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... run task successfully check_changefeed_state http://127.0.0.1:2379 f812b9b0-53bd-4346-9651-fd5739b66faf normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=f812b9b0-53bd-4346-9651-fd5739b66faf + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f812b9b0-53bd-4346-9651-fd5739b66faf -s + info='{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "normal", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' + echo '{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "normal", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' { "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "normal", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null } ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"normal",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"normal",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7362132130794692169 449348918514548740 449348918514548741 Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_changefeed_state http://127.0.0.1:2379 f812b9b0-53bd-4346-9651-fd5739b66faf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=f812b9b0-53bd-4346-9651-fd5739b66faf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c f812b9b0-53bd-4346-9651-fd5739b66faf -s # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449348916713619495" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/04/26 19:14:57.251 +08:00] [INFO] [helper.go:50] ["init log"] [file=] [level=info] [2024/04/26 19:14:57.252 +08:00] [INFO] [mem_quota.go:80] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/04/26 19:14:57.252 +08:00] [INFO] [redo.go:160] ["apply redo log starts"] [checkpointTs=449348916713619495] [resolvedTs=449348916713619495] [2024/04/26 19:14:57.252 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=ddl] [duration=397.204Β΅s] [2024/04/26 19:14:57.252 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:14:57.252 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:14:57.258 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:14:57.261 +08:00] [INFO] [mysql.go:141] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [enableOldValue=true] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.262 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [txn_cache.go:64] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitAll] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/04/26 19:14:57.263 +08:00] [INFO] [sink.go:418] ["succeed to parse parameter from sink uri"] [protocol=] [txnAtomicity=] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/04/26 19:14:57.263 +08:00] [WARN] [config.go:374] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=UTC] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/04/26 19:14:57.263 +08:00] [INFO] [worker.go:84] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/04/26 19:14:57.267 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22UTC%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/04/26 19:14:57.269 +08:00] [INFO] [file.go:283] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_1495573c-91e6-4575-b602-1a106d8b14e0_row_449348916713619500_cb1b8be6-ba70-4ede-ab87-6b377b233565.log] [endTs=449348916713619495] [2024/04/26 19:14:57.269 +08:00] [INFO] [file.go:113] ["succeed to download and sort redo logs"] [type=row] [duration=16.909345ms] [2024/04/26 19:14:57.269 +08:00] [INFO] [mysql_ddl_sink.go:100] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/04/26 19:14:57.269 +08:00] [INFO] [redo.go:216] ["apply redo log finishes"] [appliedLogCount=0] [appliedDDLCount=0] [currentCheckpoint=449348916713619495] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/04/26 19:14:57.269 +08:00] [INFO] [worker.go:99] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] Apply redo log successfully check diff successfully [Fri Apr 26 19:14:57 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 48541 + info='{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' + echo '{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null }' { "upstream_id": 7362132130794692169, "namespace": "default", "id": "f812b9b0-53bd-4346-9651-fd5739b66faf", "state": "stopped", "checkpoint_tso": 449348918514548741, "checkpoint_time": "2024-04-26 19:14:50.769", "error": null } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"stopped",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"f812b9b0-53bd-4346-9651-fd5739b66faf",' '"state":' '"stopped",' '"checkpoint_tso":' 449348918514548741, '"checkpoint_time":' '"2024-04-26' '19:14:50.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 16261bf3-4286-4f0c-9dda-06d6c319f833 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=16261bf3-4286-4f0c-9dda-06d6c319f833 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 16261bf3-4286-4f0c-9dda-06d6c319f833 -s Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + info='{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "16261bf3-4286-4f0c-9dda-06d6c319f833", "state": "normal", "checkpoint_tso": 449348920808308740, "checkpoint_time": "2024-04-26 19:14:59.519", "error": null }' + echo '{ "upstream_id": 7362132130794692169, "namespace": "default", "id": "16261bf3-4286-4f0c-9dda-06d6c319f833", "state": "normal", "checkpoint_tso": 449348920808308740, "checkpoint_time": "2024-04-26 19:14:59.519", "error": null }' { "upstream_id": 7362132130794692169, "namespace": "default", "id": "16261bf3-4286-4f0c-9dda-06d6c319f833", "state": "normal", "checkpoint_tso": 449348920808308740, "checkpoint_time": "2024-04-26 19:14:59.519", "error": null } ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"16261bf3-4286-4f0c-9dda-06d6c319f833",' '"state":' '"normal",' '"checkpoint_tso":' 449348920808308740, '"checkpoint_time":' '"2024-04-26' '19:14:59.519",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132130794692169, '"namespace":' '"default",' '"id":' '"16261bf3-4286-4f0c-9dda-06d6c319f833",' '"state":' '"normal",' '"checkpoint_tso":' 449348920808308740, '"checkpoint_time":' '"2024-04-26' '19:14:59.519",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362132130794692169 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687500cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40683, start at 2024-04-26 19:14:35.732535368 +0800 CST m=+4.356750204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: f812b9b0-53bd-4346-9651-fd5739b66faf CheckpointTs: 449348918514548741 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7362132130794692169 449348918514548740 449348920808308740 449348918514548741 run task successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: 16261bf3-4286-4f0c-9dda-06d6c319f833 CheckpointTs: 449348921909313538 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7362132130794692169 run task successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:15:06 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0a00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54709, start at 2024-04-26 19:15:05.423434369 +0800 CST m=+4.170560935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } Starting CDC state checker... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43094, start at 2024-04-26 19:14:50.366292331 +0800 CST m=+4.385909230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6877610c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43638, start at 2024-04-26 19:15:14.67098121 +0800 CST m=+4.419036848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage Starting CDC state checker... [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68750010000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40602, start at 2024-04-26 19:14:35.666332102 +0800 CST m=+4.345975218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687500cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:40683, start at 2024-04-26 19:14:35.732535368 +0800 CST m=+4.356750204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:35.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:35.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:35.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.44473.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348926206115841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348926206115841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:15:20 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.4450744509.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:23 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3d7fa4-128c-4803-98ad-0172c154298a {"id":"bd3d7fa4-128c-4803-98ad-0172c154298a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dc2de8e bd3d7fa4-128c-4803-98ad-0172c154298a /tidb/cdc/default/default/upstream/7362132572553021902 {"id":7362132572553021902,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3d7fa4-128c-4803-98ad-0172c154298a {"id":"bd3d7fa4-128c-4803-98ad-0172c154298a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dc2de8e bd3d7fa4-128c-4803-98ad-0172c154298a /tidb/cdc/default/default/upstream/7362132572553021902 {"id":7362132572553021902,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd3d7fa4-128c-4803-98ad-0172c154298a {"id":"bd3d7fa4-128c-4803-98ad-0172c154298a","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dc2de8e bd3d7fa4-128c-4803-98ad-0172c154298a /tidb/cdc/default/default/upstream/7362132572553021902 {"id":7362132572553021902,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.44561.out cli changefeed create --start-ts=449348926206115841 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: af408442-4b9e-4218-aa92-e90901ea0b78 Info: {"upstream_id":7362132572553021902,"namespace":"default","id":"af408442-4b9e-4218-aa92-e90901ea0b78","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:15:23.805733832+08:00","start_ts":449348926206115841,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348926206115841,"checkpoint_ts":449348926206115841,"checkpoint_time":"2024-04-26 19:15:20.110"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0a00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54709, start at 2024-04-26 19:15:05.423434369 +0800 CST m=+4.170560935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5800017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43023, start at 2024-04-26 19:14:50.38451618 +0800 CST m=+4.471074087 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6875e5880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:43094, start at 2024-04-26 19:14:50.366292331 +0800 CST m=+4.385909230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:50.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:50.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:50.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.46960.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348929706524673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348929706524673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449348929706524673 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:15:33 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.4700047002.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:36 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/073483da-ac71-4c52-8665-0cdb40ccaf19 {"id":"073483da-ac71-4c52-8665-0cdb40ccaf19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dfa8b90 073483da-ac71-4c52-8665-0cdb40ccaf19 /tidb/cdc/default/default/upstream/7362132639017938939 {"id":7362132639017938939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/073483da-ac71-4c52-8665-0cdb40ccaf19 {"id":"073483da-ac71-4c52-8665-0cdb40ccaf19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dfa8b90 073483da-ac71-4c52-8665-0cdb40ccaf19 /tidb/cdc/default/default/upstream/7362132639017938939 {"id":7362132639017938939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/073483da-ac71-4c52-8665-0cdb40ccaf19 {"id":"073483da-ac71-4c52-8665-0cdb40ccaf19","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1dfa8b90 073483da-ac71-4c52-8665-0cdb40ccaf19 /tidb/cdc/default/default/upstream/7362132639017938939 {"id":7362132639017938939,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449348929706524673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.47057.out cli changefeed create --start-ts=449348929706524673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362132639017938939,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:15:37.213257046+08:00","start_ts":449348929706524673,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348929706524673,"checkpoint_ts":449348929706524673,"checkpoint_time":"2024-04-26 19:15:33.463"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4881 0 --:--:-- --:--:-- --:--:-- 4911 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:15:33.463","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:15:37.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:15:33.463","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:15:37.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:15:33.463","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:15:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:15:33.463' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:15:33.463","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:15:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:15:33.463","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:15:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:15:37' + echo 'sink_checkpoint_ts is 2024-04-26' 19:15:33.463 sink_checkpoint_ts is 2024-04-26 19:15:33.463 ++ date -d '2024-04-26 19:15:33.463' +%s + checkpoint_timestamp=1714130133 ++ date -d '2024-04-26 19:15:37' +%s + current_timestamp=1714130137 + '[' 4 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6877610c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43638, start at 2024-04-26 19:15:14.67098121 +0800 CST m=+4.419036848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0ac0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54617, start at 2024-04-26 19:15:05.410106345 +0800 CST m=+4.209707592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6876d0a00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--xw4sd, pid:54709, start at 2024-04-26 19:15:05.423434369 +0800 CST m=+4.170560935 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.430 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.434 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.434 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.58813.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + tso='449348934972735489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348934972735489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687760700004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43551, start at 2024-04-26 19:15:14.592689701 +0800 CST m=+4.410000069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6877610c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--gvdvm, pid:43638, start at 2024-04-26 19:15:14.67098121 +0800 CST m=+4.419036848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:14.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:14.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:14.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.47559.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348936010039297 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348936010039297 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:15:57 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.4759747599.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:00 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 {"id":"7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e5a698f 7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 /tidb/cdc/default/default/upstream/7362132753840915410 {"id":7362132753840915410,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 {"id":"7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e5a698f 7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 /tidb/cdc/default/default/upstream/7362132753840915410 {"id":7362132753840915410,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 {"id":"7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e5a698f 7c4a7a15-ae84-4e14-bd56-84e89ffe2ba1 /tidb/cdc/default/default/upstream/7362132753840915410 {"id":7362132753840915410,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.47645.out cli changefeed create --start-ts=449348936010039297 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 03655e85-534e-470c-876e-d1107c31d163 Info: {"upstream_id":7362132753840915410,"namespace":"default","id":"03655e85-534e-470c-876e-d1107c31d163","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:16:01.180484619+08:00","start_ts":449348936010039297,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348936010039297,"checkpoint_ts":449348936010039297,"checkpoint_time":"2024-04-26 19:15:57.509"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 table row_format.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6260 0 --:--:-- --:--:-- --:--:-- 6394 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:16:36.213","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:16:30.714","now_ts":"2024-04-26 19:16:37.000","info":"The data syncing is not finished, please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:16:36.213","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:16:30.714","now_ts":"2024-04-26' '19:16:37.000","info":"The' data syncing is not finished, please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:16:36.213","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:16:30.714","now_ts":"2024-04-26' '19:16:37.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table row_format.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later table row_format.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:16:45 CST 2024] <<<<<< run test case row_format success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... [Fri Apr 26 19:16:51 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.5892958931.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:54 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3fa103-a277-45bc-8e47-45a0bafb4934 {"id":"af3fa103-a277-45bc-8e47-45a0bafb4934","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e47420f af3fa103-a277-45bc-8e47-45a0bafb4934 /tidb/cdc/default/default/upstream/7362132715732094751 {"id":7362132715732094751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3fa103-a277-45bc-8e47-45a0bafb4934 {"id":"af3fa103-a277-45bc-8e47-45a0bafb4934","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e47420f af3fa103-a277-45bc-8e47-45a0bafb4934 /tidb/cdc/default/default/upstream/7362132715732094751 {"id":7362132715732094751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af3fa103-a277-45bc-8e47-45a0bafb4934 {"id":"af3fa103-a277-45bc-8e47-45a0bafb4934","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e47420f af3fa103-a277-45bc-8e47-45a0bafb4934 /tidb/cdc/default/default/upstream/7362132715732094751 {"id":7362132715732094751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.58991.out cli changefeed create --start-ts=449348934972735489 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7362132715732094751,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:16:54.775999036+08:00","start_ts":449348934972735489,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348934972735489,"checkpoint_ts":449348934972735489,"checkpoint_time":"2024-04-26 19:15:53.552"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7362132715732094751, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:16:54.781615018+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7362132715732094751, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:16:54.781615018+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7362132715732094751, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "1970-01-01 08:00:00.000", "error": { "time": "2024-04-26T19:16:54.781615018+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7362132715732094751, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:16:54.781615018+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362132715732094751, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"1970-01-01' '08:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:16:54.781615018+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:16:56 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_sequence.finish_mark not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_sequence.finish_mark not exists for 4-th check, retry later table ddl_sequence.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e5088000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45742, start at 2024-04-26 19:17:08.268956849 +0800 CST m=+4.386387484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.258 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.258 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log table ddl_sequence.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:17:11 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e5088000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45742, start at 2024-04-26 19:17:08.268956849 +0800 CST m=+4.386387484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.258 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.258 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e50a40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45655, start at 2024-04-26 19:17:08.304055549 +0800 CST m=+4.476638069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c687e5088000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--phm32, pid:45742, start at 2024-04-26 19:17:08.268956849 +0800 CST m=+4.386387484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:08.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:08.258 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:08.258 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.49519.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348965839142913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348965839142913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:17:51 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.4955249554.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:54 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51a2d896-5fd0-4030-b6be-9f2afe859775 {"id":"51a2d896-5fd0-4030-b6be-9f2afe859775","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2013308e 51a2d896-5fd0-4030-b6be-9f2afe859775 /tidb/cdc/default/default/upstream/7362133229937253055 {"id":7362133229937253055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51a2d896-5fd0-4030-b6be-9f2afe859775 {"id":"51a2d896-5fd0-4030-b6be-9f2afe859775","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2013308e 51a2d896-5fd0-4030-b6be-9f2afe859775 /tidb/cdc/default/default/upstream/7362133229937253055 {"id":7362133229937253055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51a2d896-5fd0-4030-b6be-9f2afe859775 {"id":"51a2d896-5fd0-4030-b6be-9f2afe859775","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2013308e 51a2d896-5fd0-4030-b6be-9f2afe859775 /tidb/cdc/default/default/upstream/7362133229937253055 {"id":7362133229937253055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.49614.out cli changefeed create --start-ts=449348965839142913 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 9e49d34d-5465-4a81-8d9c-b9fbad5881e7 Info: {"upstream_id":7362133229937253055,"namespace":"default","id":"9e49d34d-5465-4a81-8d9c-b9fbad5881e7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:17:55.008610896+08:00","start_ts":449348965839142913,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449348965839142913,"checkpoint_ts":449348965839142913,"checkpoint_time":"2024-04-26 19:17:51.298"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table savepoint.finish_mark not exists for 1-th check, retry later table savepoint.finish_mark not exists for 2-th check, retry later table savepoint.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:18:55 CST 2024] <<<<<< run test case savepoint success! >>>>>> ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6157 0 --:--:-- --:--:-- --:--:-- 6314 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:18:46.363","puller_resolved_ts":"2024-04-26 19:18:39.364","last_synced_ts":"2024-04-26 19:16:30.714","now_ts":"2024-04-26 19:18:47.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:18:46.363","puller_resolved_ts":"2024-04-26' '19:18:39.364","last_synced_ts":"2024-04-26' '19:16:30.714","now_ts":"2024-04-26' '19:18:47.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 45938 7.1 0.0 13055432 151928 ? Sl 19:15 0:14 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 45997 4.0 0.0 12660936 137508 ? Sl 19:15 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-04-26T19:18:56.102+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:18:56.103+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:18:56.197+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:02.103+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:02.104+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:02.199+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } {"level":"warn","ts":"2024-04-26T19:19:06.088+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:19:06.088+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:19:06.090+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:19:06.090+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:19:06.189+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:19:06.189+0800","logger":"etcd-client","caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130346.206868,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0018e5880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130346.2069423,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130346.207879,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001988000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130346.2079575,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130346.2928257,"caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0014aefc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714130346.29289,"caller":"v3@v3.5.2/client.go:207","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":"2024-04-26T19:19:08.105+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:08.106+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:08.200+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:19:14.106+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:14.108+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:14.201+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:19:20.108+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:20.109+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:20.202+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:19:26.110+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:26.110+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:26.204+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:19:32.111+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:32.112+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:32.204+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 34 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test {"level":"warn","ts":"2024-04-26T19:19:38.113+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00111bdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:38.113+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0008e6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:19:38.205+0800","logger":"etcd-client","caller":"v3@v3.5.2/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001352540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48220, start at 2024-04-26 19:20:01.190703258 +0800 CST m=+5.282737764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.197 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48220, start at 2024-04-26 19:20:01.190703258 +0800 CST m=+5.282737764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.197 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de380017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48162, start at 2024-04-26 19:20:01.198232965 +0800 CST m=+5.379413729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.166 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.166 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6888de0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:48220, start at 2024-04-26 19:20:01.190703258 +0800 CST m=+5.282737764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:01.197 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:01.155 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:01.155 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.52011.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349011350224897 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349011350224897 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349011350224897 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:20:45 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5204752049.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:20:48 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c857ade-6432-4f30-8c2b-2378bd9523ba {"id":"7c857ade-6432-4f30-8c2b-2378bd9523ba","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22bc7d8a 7c857ade-6432-4f30-8c2b-2378bd9523ba /tidb/cdc/default/default/upstream/7362133978656357946 {"id":7362133978656357946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c857ade-6432-4f30-8c2b-2378bd9523ba {"id":"7c857ade-6432-4f30-8c2b-2378bd9523ba","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22bc7d8a 7c857ade-6432-4f30-8c2b-2378bd9523ba /tidb/cdc/default/default/upstream/7362133978656357946 {"id":7362133978656357946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c857ade-6432-4f30-8c2b-2378bd9523ba {"id":"7c857ade-6432-4f30-8c2b-2378bd9523ba","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22bc7d8a 7c857ade-6432-4f30-8c2b-2378bd9523ba /tidb/cdc/default/default/upstream/7362133978656357946 {"id":7362133978656357946,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349011350224897 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.52104.out cli changefeed create --start-ts=449349011350224897 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362133978656357946,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:20:48.720869787+08:00","start_ts":449349011350224897,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449349011350224897,"checkpoint_ts":449349011350224897,"checkpoint_time":"2024-04-26 19:20:44.909"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 51180 7.5 0.3 3168040 1444124 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 51181 8.5 0.3 3169576 1456200 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 51182 7.8 0.3 3163944 1444428 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 51184 7.5 0.3 3169060 1443284 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5321 0 --:--:-- --:--:-- --:--:-- 5400 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:21:46.709","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:21:41.609","now_ts":"2024-04-26 19:21:48.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:21:46.709","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:21:41.609","now_ts":"2024-04-26' '19:21:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:21:46.709","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:21:41.609","now_ts":"2024-04-26' '19:21:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 19816 0 --:--:-- --:--:-- --:--:-- 20083 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:21:47.709","puller_resolved_ts":"2024-04-26 19:21:47.709","last_synced_ts":"2024-04-26 19:21:41.609","now_ts":"2024-04-26 19:23:58.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:21:47.709","puller_resolved_ts":"2024-04-26' '19:21:47.709","last_synced_ts":"2024-04-26' '19:21:41.609","now_ts":"2024-04-26' '19:23:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:21:47.709","puller_resolved_ts":"2024-04-26' '19:21:47.709","last_synced_ts":"2024-04-26' '19:21:41.609","now_ts":"2024-04-26' '19:23:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898b1880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53340, start at 2024-04-26 19:24:20.49826846 +0800 CST m=+4.328216044 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898b1880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53340, start at 2024-04-26 19:24:20.49826846 +0800 CST m=+4.328216044 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898ae840024 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53246, start at 2024-04-26 19:24:20.302862069 +0800 CST m=+4.186997936 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.307 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.307 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c6898b1880019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:53340, start at 2024-04-26 19:24:20.49826846 +0800 CST m=+4.328216044 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:20.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:20.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:20.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.57274.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349078555820033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349078555820033 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349078555820033 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:25:01 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5731557317.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:04 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d75041a-1ebc-4883-a454-2055862ead26 {"id":"2d75041a-1ebc-4883-a454-2055862ead26","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26a6ea90 2d75041a-1ebc-4883-a454-2055862ead26 /tidb/cdc/default/default/upstream/7362135087186245698 {"id":7362135087186245698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d75041a-1ebc-4883-a454-2055862ead26 {"id":"2d75041a-1ebc-4883-a454-2055862ead26","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26a6ea90 2d75041a-1ebc-4883-a454-2055862ead26 /tidb/cdc/default/default/upstream/7362135087186245698 {"id":7362135087186245698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d75041a-1ebc-4883-a454-2055862ead26 {"id":"2d75041a-1ebc-4883-a454-2055862ead26","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26a6ea90 2d75041a-1ebc-4883-a454-2055862ead26 /tidb/cdc/default/default/upstream/7362135087186245698 {"id":7362135087186245698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349078555820033 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.57387.out cli changefeed create --start-ts=449349078555820033 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362135087186245698,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:25:04.998772928+08:00","start_ts":449349078555820033,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449349078555820033,"checkpoint_ts":449349078555820033,"checkpoint_time":"2024-04-26 19:25:01.278"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 57128 1.0 0.0 1967100 137296 ? Sl 19:24 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130698939851429.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 57132 0.7 0.0 2140176 120720 ? Sl 19:24 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130698942789381.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 57215 0.9 0.0 2148372 141028 ? Sl 19:24 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130698992532265.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 31990 0 --:--:-- --:--:-- --:--:-- 34714 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:26:02.178","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:25:58.178","now_ts":"2024-04-26 19:26:05.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:26:02.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:25:58.178","now_ts":"2024-04-26' '19:26:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:26:02.178","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:25:58.178","now_ts":"2024-04-26' '19:26:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5526 0 --:--:-- --:--:-- --:--:-- 5666 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:28:12.328","puller_resolved_ts":"2024-04-26 19:28:05.329","last_synced_ts":"2024-04-26 19:25:58.178","now_ts":"2024-04-26 19:28:15.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:28:12.328","puller_resolved_ts":"2024-04-26' '19:28:05.329","last_synced_ts":"2024-04-26' '19:25:58.178","now_ts":"2024-04-26' '19:28:15.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:28:12.328","puller_resolved_ts":"2024-04-26' '19:28:05.329","last_synced_ts":"2024-04-26' '19:25:58.178","now_ts":"2024-04-26' '19:28:15.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a835980017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58476, start at 2024-04-26 19:28:34.703337192 +0800 CST m=+4.312678362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a835980017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58476, start at 2024-04-26 19:28:34.703337192 +0800 CST m=+4.312678362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a8341c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58377, start at 2024-04-26 19:28:34.59233094 +0800 CST m=+4.247080436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.599 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68a835980017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:58476, start at 2024-04-26 19:28:34.703337192 +0800 CST m=+4.312678362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:30:34.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:28:34.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:18:34.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.62511.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349145983713281 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349145983713281 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349145983713281 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:29:18 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.6254962551.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:29:21 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af90d4a-d376-4828-8ac3-1b612ecf584d {"id":"9af90d4a-d376-4828-8ac3-1b612ecf584d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8f8e90 9af90d4a-d376-4828-8ac3-1b612ecf584d /tidb/cdc/default/default/upstream/7362136190620545819 {"id":7362136190620545819,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af90d4a-d376-4828-8ac3-1b612ecf584d {"id":"9af90d4a-d376-4828-8ac3-1b612ecf584d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8f8e90 9af90d4a-d376-4828-8ac3-1b612ecf584d /tidb/cdc/default/default/upstream/7362136190620545819 {"id":7362136190620545819,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9af90d4a-d376-4828-8ac3-1b612ecf584d {"id":"9af90d4a-d376-4828-8ac3-1b612ecf584d","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2a8f8e90 9af90d4a-d376-4828-8ac3-1b612ecf584d /tidb/cdc/default/default/upstream/7362136190620545819 {"id":7362136190620545819,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349145983713281 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.62611.out cli changefeed create --start-ts=449349145983713281 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362136190620545819,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:29:22.181901432+08:00","start_ts":449349145983713281,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449349145983713281,"checkpoint_ts":449349145983713281,"checkpoint_time":"2024-04-26 19:29:18.495"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 19022 0 --:--:-- --:--:-- --:--:-- 19540 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:29:18.495","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:29:42.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:29:18.495","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:29:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:29:18.495","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:29:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri Apr 26 19:29:52 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Fri Apr 26 19:29:52 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade5f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63691, start at 2024-04-26 19:30:07.916634062 +0800 CST m=+4.550858120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade5f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63691, start at 2024-04-26 19:30:07.916634062 +0800 CST m=+4.550858120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade6e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63598, start at 2024-04-26 19:30:07.96309021 +0800 CST m=+4.664809567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.928 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.928 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68ade5f8001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:63691, start at 2024-04-26 19:30:07.916634062 +0800 CST m=+4.550858120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:07.924 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:07.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:07.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.67744.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349185826979841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349185826979841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:31:50 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.6779067792.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:53 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c325ddc2-9121-4c1d-9734-bc93fd1c52d1 {"id":"c325ddc2-9121-4c1d-9734-bc93fd1c52d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bfc5e10 c325ddc2-9121-4c1d-9734-bc93fd1c52d1 /tidb/cdc/default/default/upstream/7362136585372395897 {"id":7362136585372395897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c325ddc2-9121-4c1d-9734-bc93fd1c52d1 {"id":"c325ddc2-9121-4c1d-9734-bc93fd1c52d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bfc5e10 c325ddc2-9121-4c1d-9734-bc93fd1c52d1 /tidb/cdc/default/default/upstream/7362136585372395897 {"id":7362136585372395897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c325ddc2-9121-4c1d-9734-bc93fd1c52d1 {"id":"c325ddc2-9121-4c1d-9734-bc93fd1c52d1","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2bfc5e10 c325ddc2-9121-4c1d-9734-bc93fd1c52d1 /tidb/cdc/default/default/upstream/7362136585372395897 {"id":7362136585372395897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.67850.out cli changefeed create --start-ts=449349185826979841 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 8eb0b4ad-6dcf-4d40-ba54-13b52a83c0b8 Info: {"upstream_id":7362136585372395897,"namespace":"default","id":"8eb0b4ad-6dcf-4d40-ba54-13b52a83c0b8","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:31:54.195673152+08:00","start_ts":449349185826979841,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449349185826979841,"checkpoint_ts":449349185826979841,"checkpoint_time":"2024-04-26 19:31:50.485"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "mysql.host"="127.0.0.1" "threadcount"="2" "updateproportion"="0" "dotransactions"="false" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "readproportion"="0" "mysql.db"="testSync" "requestdistribution"="uniform" "mysql.port"="4000" "scanproportion"="0" "recordcount"="10" "workload"="core" "insertproportion"="0" ********************************************** Run finished, takes 9.206659ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2188.1, Avg(us): 1760, Min(us): 874, Max(us): 4474, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "recordcount"="10" "dotransactions"="false" "mysql.port"="4000" "threadcount"="2" "workload"="core" "mysql.user"="root" "readallfields"="true" "mysql.host"="127.0.0.1" "scanproportion"="0" "operationcount"="0" "mysql.db"="testSync" "insertproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 4.078798ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3516.8, Avg(us): 725, Min(us): 483, Max(us): 1436, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "operationcount"="0" "mysql.user"="root" "readproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "mysql.db"="testSync" "threadcount"="2" "dotransactions"="false" "readallfields"="true" "insertproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 3.848561ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3704.9, Avg(us): 657, Min(us): 430, Max(us): 1445, 95th(us): 2000, 99th(us): 2000 table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists ____________________________________ *************************** 1. row *************************** primary_ts: 449349185826979841 secondary_ts: 449349188171857933 *************************** 2. row *************************** primary_ts: 449349193691299840 secondary_ts: 449349193807953921 *************************** 3. row *************************** primary_ts: 449349201555619840 secondary_ts: 449349201698488321 *************************** 4. row *************************** primary_ts: 449349209419939840 secondary_ts: 449349209562808321 *************************** 5. row *************************** primary_ts: 449349217284259840 secondary_ts: 449349217427128321 *************************** 6. row *************************** primary_ts: 449349225148579840 secondary_ts: 449349225317400577 *************************** 7. row *************************** primary_ts: 449349233012899840 secondary_ts: 449349233181720577 *************************** 8. row *************************** primary_ts: 449349240877219840 secondary_ts: 449349241046302721 *************************** 9. row *************************** primary_ts: 449349248741539840 secondary_ts: 449349248936574977 skip invalid syncpoint primary_ts: 449349185826979841, first_ddl_ts: 449349188605706253 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:35:54 CST 2024] <<<<<< run test case syncpoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c475f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70155, start at 2024-04-26 19:36:17.547930987 +0800 CST m=+4.079904339 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c475f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70155, start at 2024-04-26 19:36:17.547930987 +0800 CST m=+4.079904339 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c4754c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70067, start at 2024-04-26 19:36:17.518310574 +0800 CST m=+4.110198395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.491 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68c475f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:70155, start at 2024-04-26 19:36:17.547930987 +0800 CST m=+4.079904339 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:38:17.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:36:17.534 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:26:17.534 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.74181.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349267026870273 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349267026870273 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:37:00 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.7422574227.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:37:03 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f246157-d545-456d-a475-adc72c630710 {"id":"7f246157-d545-456d-a475-adc72c630710","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a319f598e 7f246157-d545-456d-a475-adc72c630710 /tidb/cdc/default/default/upstream/7362138174627663522 {"id":7362138174627663522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f246157-d545-456d-a475-adc72c630710 {"id":"7f246157-d545-456d-a475-adc72c630710","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a319f598e 7f246157-d545-456d-a475-adc72c630710 /tidb/cdc/default/default/upstream/7362138174627663522 {"id":7362138174627663522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f246157-d545-456d-a475-adc72c630710 {"id":"7f246157-d545-456d-a475-adc72c630710","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a319f598e 7f246157-d545-456d-a475-adc72c630710 /tidb/cdc/default/default/upstream/7362138174627663522 {"id":7362138174627663522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1651 --:--:-- --:--:-- --:--:-- 1670 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 build mysql_test successfully run all mysql test cases time="2024-04-26T19:37:23+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-04-26T19:37:23+08:00" level=warning msg="Create new db&{0 0xc000014098 0 {0 0} [0xc000116000] map[] 0 1 0xc000038240 false map[0xc000116000:map[0xc000116000:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/alter_table.test: ok! 70 test cases passed, take time 6.364359044 s time="2024-04-26T19:38:04+08:00" level=info msg="run test [alter_table] ok" time="2024-04-26T19:38:04+08:00" level=warning msg="Create new db&{0 0xc00030e160 0 {0 0} [0xc000210990] map[] 0 1 0xc0003187e0 false map[0xc000210990:map[0xc000210990:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/bigint.test: ok! 138 test cases passed, take time 4.9159583829999995 s time="2024-04-26T19:38:09+08:00" level=info msg="run test [bigint] ok" time="2024-04-26T19:38:09+08:00" level=warning msg="Create new db&{0 0xc00030e720 0 {0 0} [0xc00044e5a0] map[] 0 1 0xc0003199e0 false map[0xc00044e5a0:map[0xc00044e5a0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/composite_index.test: ok! 12 test cases passed, take time 0.246441871 s time="2024-04-26T19:38:09+08:00" level=info msg="run test [composite_index] ok" time="2024-04-26T19:38:09+08:00" level=warning msg="Create new db&{0 0xc00040a020 0 {0 0} [0xc000210240] map[] 0 1 0xc000038420 false map[0xc000210240:map[0xc000210240:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 0.444434972 s time="2024-04-26T19:38:10+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-04-26T19:38:10+08:00" level=warning msg="Create new db&{0 0xc00040a0c0 0 {0 0} [0xc0002103f0] map[] 0 1 0xc0000386c0 false map[0xc0002103f0:map[0xc0002103f0:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/date_formats.test: ok! 99 test cases passed, take time 0.972110773 s time="2024-04-26T19:38:11+08:00" level=info msg="run test [date_formats] ok" time="2024-04-26T19:38:11+08:00" level=warning msg="Create new db&{0 0xc00040a400 0 {0 0} [0xc000211a70] map[] 0 1 0xc0000395c0 false map[0xc000211a70:map[0xc000211a70:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/drop.test: ok! 48 test cases passed, take time 1.919063488 s time="2024-04-26T19:38:13+08:00" level=info msg="run test [drop] ok" time="2024-04-26T19:38:13+08:00" level=warning msg="Create new db&{0 0xc00030e828 0 {0 0} [0xc00044f050] map[] 0 1 0xc0002ee480 false map[0xc00044f050:map[0xc00044f050:true]] map[] 0 0 0 0 0 0 0 0 0x48dd80}" ./t/mysql_replace.test: ok! 16 test cases passed, take time 0.19093113 s time="2024-04-26T19:38:13+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later table test.finish_mark not exists for 9-th check, retry later table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later table test.finish_mark not exists for 12-th check, retry later table test.finish_mark not exists for 13-th check, retry later table test.finish_mark not exists for 14-th check, retry later table test.finish_mark not exists for 15-th check, retry later table test.finish_mark not exists for 16-th check, retry later table test.finish_mark not exists for 17-th check, retry later table test.finish_mark not exists for 18-th check, retry later table test.finish_mark not exists for 19-th check, retry later table test.finish_mark not exists for 20-th check, retry later table test.finish_mark not exists for 21-th check, retry later table test.finish_mark not exists for 22-th check, retry later table test.finish_mark not exists for 23-th check, retry later table test.finish_mark not exists for 24-th check, retry later table test.finish_mark not exists for 25-th check, retry later table test.finish_mark not exists for 26-th check, retry later table test.finish_mark not exists for 27-th check, retry later table test.finish_mark not exists for 28-th check, retry later table test.finish_mark not exists for 29-th check, retry later table test.finish_mark not exists for 30-th check, retry later table test.finish_mark not exists for 31-th check, retry later table test.finish_mark not exists for 32-th check, retry later table test.finish_mark not exists for 33-th check, retry later table test.finish_mark not exists for 34-th check, retry later table test.finish_mark not exists for 35-th check, retry later table test.finish_mark not exists for 36-th check, retry later table test.finish_mark not exists for 37-th check, retry later table test.finish_mark not exists for 38-th check, retry later table test.finish_mark not exists for 39-th check, retry later table test.finish_mark not exists for 40-th check, retry later table test.finish_mark not exists for 41-th check, retry later table test.finish_mark not exists for 42-th check, retry later table test.finish_mark not exists for 43-th check, retry later table test.finish_mark not exists for 44-th check, retry later table test.finish_mark not exists for 45-th check, retry later table test.finish_mark not exists for 46-th check, retry later table test.finish_mark not exists for 47-th check, retry later table test.finish_mark not exists for 48-th check, retry later table test.finish_mark not exists for 49-th check, retry later table test.finish_mark not exists for 50-th check, retry later table test.finish_mark not exists for 51-th check, retry later table test.finish_mark not exists for 52-th check, retry later table test.finish_mark not exists for 53-th check, retry later table test.finish_mark not exists for 54-th check, retry later table test.finish_mark not exists for 55-th check, retry later table test.finish_mark not exists for 56-th check, retry later table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:40:07 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3ce8c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77110, start at 2024-04-26 19:40:28.963703558 +0800 CST m=+4.612189966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v6.5.8-7-g0be447c11 Edition: Community Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa Git Branch: HEAD UTC Build Time: 2024-03-30 14:02:19 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a Git Commit Branch: HEAD UTC Build Time: 2024-03-30 14:05:17 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log Starting CDC state checker... # github.com/pingcap/tiflow/tests/utils/cdc_state_checker ./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState) *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) ./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement: *cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method) start tidb cluster failed The 2 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3ce8c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77110, start at 2024-04-26 19:40:28.963703558 +0800 CST m=+4.612189966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Starting Downstream PD... Release Version: v6.5.9-1-g0ecafcb36 Edition: Community Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371 Git Branch: release-6.5 UTC Build Time: 2024-04-26 06:25:13 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 6.5.9 Edition: Community Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5 Git Commit Branch: release-6.5 UTC Build Time: 2024-04-25 09:35:48 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v6.5.9-3-g7357987916 Edition: Community Git Commit Hash: 7357987916b70e4721952d8998428db7bfb78adc Git Branch: release-6.5 UTC Build Time: 2024-04-17 15:20:26 GoVersion: go1.19.13 Race Enabled: false TiKV Min Version: 6.2.0-alpha Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3cc500004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77023, start at 2024-04-26 19:40:28.823168889 +0800 CST m=+4.571158579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 110 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. tikv_gc_leader_uuid 63c68d3ce8c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-164--zc3dq, pid:77110, start at 2024-04-26 19:40:28.963703558 +0800 CST m=+4.612189966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:28.970 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:28.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:28.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.81039.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349333210365953 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349333210365953 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:41:12 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.8108081082.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:41:16 GMT < Content-Length: 612 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afce39f7-6b22-4043-aefa-299a4c7d241e {"id":"afce39f7-6b22-4043-aefa-299a4c7d241e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3575a790 afce39f7-6b22-4043-aefa-299a4c7d241e /tidb/cdc/default/default/upstream/7362139255416327829 {"id":7362139255416327829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afce39f7-6b22-4043-aefa-299a4c7d241e {"id":"afce39f7-6b22-4043-aefa-299a4c7d241e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3575a790 afce39f7-6b22-4043-aefa-299a4c7d241e /tidb/cdc/default/default/upstream/7362139255416327829 {"id":7362139255416327829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afce39f7-6b22-4043-aefa-299a4c7d241e {"id":"afce39f7-6b22-4043-aefa-299a4c7d241e","address":"127.0.0.1:8300","version":"v6.5.9-1-gc3f88235b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3575a790 afce39f7-6b22-4043-aefa-299a4c7d241e /tidb/cdc/default/default/upstream/7362139255416327829 {"id":7362139255416327829,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.81147.out cli changefeed create --start-ts=449349333210365953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: c2936e6b-ade8-4d50-bd81-349a0d1365d6 Info: {"upstream_id":7362139255416327829,"namespace":"default","id":"c2936e6b-ade8-4d50-bd81-349a0d1365d6","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:41:16.396191879+08:00","start_ts":449349333210365953,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-1-gc3f88235b","resolved_ts":449349333210365953,"checkpoint_ts":449349333210365953,"checkpoint_time":"2024-04-26 19:41:12.708"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri Apr 26 19:42:13 CST 2024] <<<<<< run test case tiflash success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-6.5-pull_cdc_integration_test-164/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE