Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-lntbn-50k3r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1fa8de8d6a30d85a371744e502bc05d132ecdadc" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-lntbn" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=1183f660-4382-4fae-8a39-a120ac58d09f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-lntbn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-d4c91f37a2cd54d8a1ba68aedcc7e1b173b616003bd6c892b8b1a5e40c8eb9f7 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-lntbn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3301327150=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 bash' debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-h9193 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 46703 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: d38714d3612626f7 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 22 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 07:05:28 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11000/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11000/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11000/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11000/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [pkg/sink/codec/avro/avro.go] [Pipeline] echo diff file not matched: pkg/sink/codec/avro/avro.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-fd3a6e7-b417a77) 202276864 bytes in 3.64 secs (55549059 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/.git/ .git HEAD is now at f21441d0a Merge commit 'b417a77d3bfa2c9235cbca0d58dab40f95c1ca50' into HEAD POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/11000/head -> origin/pr/11000/head Warning: you are leaving 1 commit behind, not connected to any of your branches: f21441d0a Merge commit 'b417a77d3bfa2c9235cbca0d58dab40f95c1ca50' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch f21441d0a HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) ๐Ÿšง Checkouting to base SHA:fd3a6e756d0d488c7e60bedb05e21e42320c3dc7... HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) 47417d775 mounter(ticdc): calculate row level checksum for timestmap by using UTC time zone (#10564) (#10646) 3ab47635f *: release-7.5 bump tidb dependencies (#10970) ๐Ÿšง Pre-merge heads of pull requests to base SHA: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 ... Merge made by the 'ort' strategy. pkg/sink/codec/avro/avro.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) ๐Ÿงพ Pre-merged result: 319e94f03ea7eae9ae289383928b8658e802a9a3 319e94f03 Merge commit 'b417a77d3bfa2c9235cbca0d58dab40f95c1ca50' into HEAD b417a77d3 fix tests. fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-fd3a6e7-b417a77 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh release-7.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 757 0 --:--:-- --:--:-- --:--:-- 759 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 831 0 --:--:-- --:--:-- --:--:-- 836 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 521 0 --:--:-- --:--:-- --:--:-- 525 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 886 0 --:--:-- --:--:-- --:--:-- 891 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/61c4bd61254323d2dc68a8de048000ea036e67ac/centos7/tidb-server.tar.gz 2024-05-06 15:06:07 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/61c4bd61254323d2dc68a8de048000ea036e67ac/centos7/tidb-server.tar.gz [511672373/511672373] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz 2024-05-06 15:06:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz [173223227/173223227] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz 2024-05-06 15:07:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz [810269604/810269604] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz 2024-05-06 15:07:34 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz [263167205/263167205] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 15:07:40 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 15:07:42 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 15:07:43 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 15:07:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 15:07:49 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 15:08:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 15:08 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 15:08 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 15:07 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 15:07 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 1 10:52 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 1 23:57 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 15:08 . drwxr-sr-x 3 jenkins jenkins 4.0K May 6 15:08 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 15:07 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 15:07 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 1 10:52 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 1 23:57 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/cdc-integration-test/rev-fd3a6e7-b417a77) 1137768448 bytes in 1.84 secs (619734003 bytes/sec) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 1.1G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 15:08 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 15:08 .. -rwxr-xr-x 1 jenkins jenkins 204M May 6 11:09 cdc -rwxr-xr-x 1 jenkins jenkins 339M May 6 11:08 cdc.test -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_storage_consumer + '[' -f ./bin/cdc ']' + '[' -f ./bin/cdc_kafka_consumer ']' + '[' -f ./bin/cdc_storage_consumer ']' + '[' -f ./bin/cdc.test ']' + ls -alh ./bin total 1.1G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 15:08 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 15:08 .. -rwxr-xr-x 1 jenkins jenkins 204M May 6 11:09 cdc -rwxr-xr-x 1 jenkins jenkins 339M May 6 11:08 cdc.test -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_storage_consumer + ./bin/cdc version Release Version: v7.5.1-23-gf21441d0a Git Commit Hash: f21441d0a8b3471f8548ffca9a37735214c532e0 Git Branch: HEAD UTC Build Time: 2024-05-06 03:04:00 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-fd3a6e7-b417a77 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 15:08 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 15:08 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 6 15:08 bin -rwxr-xr-x 1 jenkins jenkins 204M May 6 11:09 cdc -rwxr-xr-x 1 jenkins jenkins 339M May 6 11:08 cdc.test -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K May 6 15:08 etc -rwxr-xr-x 1 jenkins jenkins 17M May 6 15:08 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 15:08 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 15:08 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 6 15:08 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 15:08 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 6 15:08 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 15:08 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 6 15:08 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 15:08 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 15:08 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 15:08 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M May 6 15:08 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 6 15:08 minio -rwxr-xr-x 1 jenkins jenkins 33M May 6 15:08 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M May 6 15:08 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M May 6 15:08 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M May 6 15:08 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M May 6 15:08 pd-server -rwxr-xr-x 1 jenkins jenkins 25M May 6 15:08 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M May 6 15:08 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 6 15:08 share -rwxr-xr-x 1 jenkins jenkins 29M May 6 15:08 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M May 6 15:08 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 6 15:08 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M May 6 15:08 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 6 15:08 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 67.17 secs (50099024 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-1kpq7-cgk72 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "047ba8233dbb44f2f9e0edaade6fb519f5593633" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-1kpq7" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] node [Pipeline] { [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@156b79ff; decorates RemoteLauncher[hudson.remoting.Channel@25fef051:JNLP4-connect connection from 10.233.66.158/10.233.66.158:42316] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xmbmn-nl61z --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ec638c613aea3494e1a103c6dccccdd20f3ff0d0" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xmbmn" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] checkout The recommended git tool is: git [Pipeline] podTemplate [Pipeline] { Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] node [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3ff17ccf; decorates RemoteLauncher[hudson.remoting.Channel@2226a61e:JNLP4-connect connection from 10.233.106.213/10.233.106.213:33052] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-pl2f7-d469n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d3f53141f8e8d51f5835864f12608c56cf1db70f" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-pl2f7" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26fโ€™ is offline Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-n19r9-ccx35 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9043deb163fc68778b6a64f7a9e522c9159abb2d" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-n19r9" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xpbw3-9fq5d --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "daafddee454b7eee963b275ff7f2847ba63ae4b2" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xpbw3" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-wdwqq-5n29k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "db0a07acd2bd782d310bb0b31ce7c2bfc54a2bfa" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-wdwqq" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-qqf95-fck7w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "027972f6b671a9dc979419288b790723a94c9ef4" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-qqf95" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-88wx8-hh5n9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c32808b2f1e6f555a6542d9327ce533d77e8ad9b" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-88wx8" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-j01wm-k5fz8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3a9b5a6bba5c1f24985608b8d887fa7e61685df0" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-j01wm" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ff23f-lj2fj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "23c014a29692703f6c9805de13e2e7d7d824e271" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ff23f" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5โ€™ is offline Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-x30bf-v43d2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a7281d928758421ff81463a6583795e35a36d3a8" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-x30bf" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-jktf9-rh8zn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4a025ec7c465d91fd701856318b7fcee8654deef" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-jktf9" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7โ€™ is not accepting tasks โ€˜pingcap-docs-cn-pull-verify-2723-4tgl6-splrv-89h35โ€™ doesnโ€™t have label โ€˜ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8โ€™ โ€˜pingcap-docs-cn-pull-verify-2723-zgfld-192j2-fx4fwโ€™ doesnโ€™t have label โ€˜ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8โ€™ Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8-fdjvw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "849b4e68a4a8eb5e526460ea1699aa4e6267c380" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 20.54 secs (163813330 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro lossy_ddl storage_csv_update PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=d1cd0702-df87-4ed0-849c-31e9bb490b38 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-1kpq7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-1kpq7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] { [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1daea5fe; decorates RemoteLauncher[hudson.remoting.Channel@4171bba3:JNLP4-connect connection from 10.233.107.192/10.233.107.192:46880] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@76a82e45; decorates RemoteLauncher[hudson.remoting.Channel@53b69fa0:JNLP4-connect connection from 10.233.106.67/10.233.106.67:32904] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@50b839ba; decorates RemoteLauncher[hudson.remoting.Channel@45d19400:JNLP4-connect connection from 10.233.67.92/10.233.67.92:38402] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@64d43f37; decorates RemoteLauncher[hudson.remoting.Channel@13c0e49c:JNLP4-connect connection from 10.233.107.30/10.233.107.30:47940] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3bbb3b16; decorates RemoteLauncher[hudson.remoting.Channel@62d0507:JNLP4-connect connection from 10.233.97.133/10.233.97.133:50278] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3fac5eb9; decorates RemoteLauncher[hudson.remoting.Channel@40aef0fd:JNLP4-connect connection from 10.233.100.103/10.233.100.103:46862] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3bec30b2; decorates RemoteLauncher[hudson.remoting.Channel@3d4c8fca:JNLP4-connect connection from 10.233.66.205/10.233.66.205:33650] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@157a5abc; decorates RemoteLauncher[hudson.remoting.Channel@4bf56425:JNLP4-connect connection from 10.233.66.13/10.233.66.13:41022] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@278d2482; decorates RemoteLauncher[hudson.remoting.Channel@27de2b89:JNLP4-connect connection from 10.233.68.232/10.233.68.232:34870] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1e09c2; decorates RemoteLauncher[hudson.remoting.Channel@5d150554:JNLP4-connect connection from 10.233.67.14/10.233.67.14:34254] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@77997da3; decorates RemoteLauncher[hudson.remoting.Channel@75bd983c:JNLP4-connect connection from 10.233.66.238/10.233.66.238:33820] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-h7j4j-l3zjt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ff9785152788f62450f93764fb0f138e5ab8eeb0" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-h7j4j" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-bvb0p-8kg88 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7cfca4b77e84090d69ce3986ce4edc351161a6e1" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-bvb0p" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k2146-mdr5n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "dfb40667582f7a0ea665fcfa20746f023d3903bb" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k2146" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k1v9z-rzphq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "478fa449656c3383053b8469ed29494ab89af72f" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k1v9z" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Starting Upstream TiDB... Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-7dps8-0k0z4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7dbfc92fd0f1d0797e2c38d369fecbe9a4820e14" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-7dps8" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-cn3gg-lms8j --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "aa8aa471d349951398ee35603d1823811f6a3952" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-cn3gg" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f4653c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:1236, start at 2024-05-06 15:10:12.316127923 +0800 CST m=+5.129007620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:12.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:12.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:12.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f4653c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:1236, start at 2024-05-06 15:10:12.316127923 +0800 CST m=+5.129007620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:12.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:12.303 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:12.303 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f465e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:1315, start at 2024-05-06 15:10:12.382844758 +0800 CST m=+5.123818059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:12.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:12.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:12.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-c7g74-d372c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ad9b3bff614a7714a7833b0e8618331c15473f66" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-c7g74" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-gx8nm-rxmrn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "84ca90101f8c588002cd5c8f87292d9d2d304a95" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-gx8nm" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Still waiting to schedule task Waiting for next available executor on โ€˜ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55cโ€™ [Mon May 6 15:10:15 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.25622564.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-hvd30-lm68b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1f263a53d57129d986652de756e7ef28d495ec89" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-hvd30" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:10:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d658980d-b4cc-4f57-b897-9bd65d626577 {"id":"d658980d-b4cc-4f57-b897-9bd65d626577","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbcf74bf2 d658980d-b4cc-4f57-b897-9bd65d626577 /tidb/cdc/default/default/upstream/7365780453545684305 {"id":7365780453545684305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d658980d-b4cc-4f57-b897-9bd65d626577 {"id":"d658980d-b4cc-4f57-b897-9bd65d626577","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbcf74bf2 d658980d-b4cc-4f57-b897-9bd65d626577 /tidb/cdc/default/default/upstream/7365780453545684305 {"id":7365780453545684305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d658980d-b4cc-4f57-b897-9bd65d626577 {"id":"d658980d-b4cc-4f57-b897-9bd65d626577","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbcf74bf2 d658980d-b4cc-4f57-b897-9bd65d626577 /tidb/cdc/default/default/upstream/7365780453545684305 {"id":7365780453545684305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:10:18 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26082610.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:10:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaaa79e-e97e-486f-ad0b-4ef2f7134992 {"id":"4aaaa79e-e97e-486f-ad0b-4ef2f7134992","address":"127.0.0.1:8400","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4cbcf7761e 4aaaa79e-e97e-486f-ad0b-4ef2f7134992 /tidb/cdc/default/default/upstream/7365780455419086363 {"id":7365780455419086363,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaaa79e-e97e-486f-ad0b-4ef2f7134992 {"id":"4aaaa79e-e97e-486f-ad0b-4ef2f7134992","address":"127.0.0.1:8400","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4cbcf7761e 4aaaa79e-e97e-486f-ad0b-4ef2f7134992 /tidb/cdc/default/default/upstream/7365780455419086363 {"id":7365780455419086363,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4aaaa79e-e97e-486f-ad0b-4ef2f7134992 {"id":"4aaaa79e-e97e-486f-ad0b-4ef2f7134992","address":"127.0.0.1:8400","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4cbcf7761e 4aaaa79e-e97e-486f-ad0b-4ef2f7134992 /tidb/cdc/default/default/upstream/7365780455419086363 {"id":7365780455419086363,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table bdr_mode.t1 exists table bdr_mode.t1 exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2670.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365780453545684305,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-06T15:10:23.988405254+08:00","start_ts":449571566086258691,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571566086258691,"checkpoint_ts":449571566086258691,"checkpoint_time":"2024-05-06 15:10:23.852"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 20.70 secs (162556239 bytes/sec) [Pipeline] { + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2705.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml [Pipeline] sh Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365780455419086363,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-06T15:10:25.843615995+08:00","start_ts":449571566569127940,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571566569127940,"checkpoint_ts":449571566569127940,"checkpoint_time":"2024-05-06 15:10:25.694"} PASS [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=a6e5578f-5a4a-4fd9-b000-25f2eb5b9945 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xmbmn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xmbmn ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] container The recommended git tool is: git [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e7d330f; decorates RemoteLauncher[hudson.remoting.Channel@54bc17e2:JNLP4-connect connection from 10.233.88.250/10.233.88.250:53510] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@80e51f8; decorates RemoteLauncher[hudson.remoting.Channel@ecce696:JNLP4-connect connection from 10.233.67.208/10.233.67.208:39924] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] timeout Cloning the remote Git repository Using shallow clone with depth 1 Timeout set to expire in 40 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6d370d36; decorates RemoteLauncher[hudson.remoting.Channel@4820d721:JNLP4-connect connection from 10.233.88.249/10.233.88.249:34910] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + set +x [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@12564bf5; decorates RemoteLauncher[hudson.remoting.Channel@64ff41d2:JNLP4-connect connection from 10.233.68.192/10.233.68.192:44006] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Cloning repository https://github.com/PingCAP-QE/ci.git Timeout set to expire in 40 min [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@55be903; decorates RemoteLauncher[hudson.remoting.Channel@3fdb7d1a:JNLP4-connect connection from 10.233.86.220/10.233.86.220:39030] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4efb51d3; decorates RemoteLauncher[hudson.remoting.Channel@3072f56b:JNLP4-connect connection from 10.233.69.143/10.233.69.143:51958] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5ae7217a; decorates RemoteLauncher[hudson.remoting.Channel@1595c571:JNLP4-connect connection from 10.233.69.157/10.233.69.157:48442] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@13dcbb1d; decorates RemoteLauncher[hudson.remoting.Channel@295a19ec:JNLP4-connect connection from 10.233.86.180/10.233.86.180:51078] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@27112767; decorates RemoteLauncher[hudson.remoting.Channel@42140a73:JNLP4-connect connection from 10.233.105.248/10.233.105.248:39614] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" check diff successfully Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 wait process cdc.test exit for 1-th time... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) wait process cdc.test exit for 2-th time... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:10:32 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f614bc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:1133, start at 2024-05-06 15:10:39.922903558 +0800 CST m=+5.417739545 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:39.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:39.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:39.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f614bc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:1133, start at 2024-05-06 15:10:39.922903558 +0800 CST m=+5.417739545 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:39.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:39.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:39.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f616300009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:1222, start at 2024-05-06 15:10:40.020532488 +0800 CST m=+5.458589670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:40.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:40.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:40.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Mon May 6 15:10:44 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.25952597.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:10:47 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58de076a-7427-4fc4-b199-3546f6dc4837 {"id":"58de076a-7427-4fc4-b199-3546f6dc4837","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c22f0 58de076a-7427-4fc4-b199-3546f6dc4837 /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58de076a-7427-4fc4-b199-3546f6dc4837 {"id":"58de076a-7427-4fc4-b199-3546f6dc4837","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c22f0 58de076a-7427-4fc4-b199-3546f6dc4837 /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58de076a-7427-4fc4-b199-3546f6dc4837 {"id":"58de076a-7427-4fc4-b199-3546f6dc4837","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c22f0 58de076a-7427-4fc4-b199-3546f6dc4837 /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "mysql.db"="consistent_replicate_ddl" "threadcount"="10" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "fieldcount"="3" "insertproportion"="0" "updateproportion"="0" "operationcount"="0" "readproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "readallfields"="true" "workload"="core" "recordcount"="50" ********************************************** Run finished, takes 12.988718ms INSERT - Takes(s): 0.0, Count: 50, OPS: 7039.5, Avg(us): 2291, Min(us): 923, Max(us): 6243, 95th(us): 7000, 99th(us): 7000 table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 17.98 secs (187153627 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f6fd900013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:3695, start at 2024-05-06 15:10:54.851192882 +0800 CST m=+5.043814437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:54.858 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:54.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:54.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f6fd900013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:3695, start at 2024-05-06 15:10:54.851192882 +0800 CST m=+5.043814437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:54.858 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:54.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:54.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32f6fe240016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:3771, start at 2024-05-06 15:10:54.892401788 +0800 CST m=+5.023043128 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:12:54.898 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:10:54.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:00:54.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later [Mon May 6 15:10:58 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.51375139.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:11:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:11:01 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.51815183.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:11:04 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc615dc2-cd29-46f0-80ae-b322e1d9973a {"id":"cc615dc2-cd29-46f0-80ae-b322e1d9973a","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97ea1a cc615dc2-cd29-46f0-80ae-b322e1d9973a /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc615dc2-cd29-46f0-80ae-b322e1d9973a {"id":"cc615dc2-cd29-46f0-80ae-b322e1d9973a","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97ea1a cc615dc2-cd29-46f0-80ae-b322e1d9973a /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc615dc2-cd29-46f0-80ae-b322e1d9973a {"id":"cc615dc2-cd29-46f0-80ae-b322e1d9973a","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f6879667-8f02-43f0-bb4c-d15d6b5d9a73 {"id":"f6879667-8f02-43f0-bb4c-d15d6b5d9a73","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97e9ed f6879667-8f02-43f0-bb4c-d15d6b5d9a73 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd97ea1a cc615dc2-cd29-46f0-80ae-b322e1d9973a /tidb/cdc/default/default/upstream/7365780635885604423 {"id":7365780635885604423,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 13.60 secs (247437865 bytes/sec) [Pipeline] { [Pipeline] cache table consistent_replicate_ddl.usertable_bak exists table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:11:13 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.28252827.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table capture_suicide_while_balance_table.t4 exists % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 1127 3382 --:--:-- --:--:-- --:--:-- 3500 { "status": true, "message": "" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:11:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {UpstreamID:7365780572903040080 Namespace:default ID:9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:10:48.137501553 +0800 CST StartTs:449571572421754889 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00433b050 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571572447707139} {CheckpointTs:449571579643559943 MinTableBarrierTs:449571579643559943 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571579643559943, checkpointTs: 449571579643559943, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/784eac33-2618-482f-8967-7501f5638dcc {"id":"784eac33-2618-482f-8967-7501f5638dcc","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c23ba 784eac33-2618-482f-8967-7501f5638dcc /tidb/cdc/default/default/changefeed/info/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"upstream-id":7365780572903040080,"namespace":"default","changefeed-id":"9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:10:48.137501553+08:00","start-ts":449571572421754889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571572447707139} /tidb/cdc/default/default/changefeed/status/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":449571579643559943,"min-table-barrier-ts":449571579643559943,"admin-job-type":0} /tidb/cdc/default/default/task/position/784eac33-2618-482f-8967-7501f5638dcc/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {UpstreamID:7365780572903040080 Namespace:default ID:9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:10:48.137501553 +0800 CST StartTs:449571572421754889 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00433b050 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571572447707139} {CheckpointTs:449571579643559943 MinTableBarrierTs:449571579643559943 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571579643559943, checkpointTs: 449571579643559943, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/784eac33-2618-482f-8967-7501f5638dcc {"id":"784eac33-2618-482f-8967-7501f5638dcc","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c23ba 784eac33-2618-482f-8967-7501f5638dcc /tidb/cdc/default/default/changefeed/info/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"upstream-id":7365780572903040080,"namespace":"default","changefeed-id":"9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:10:48.137501553+08:00","start-ts":449571572421754889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571572447707139} /tidb/cdc/default/default/changefeed/status/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":449571579643559943,"min-table-barrier-ts":449571579643559943,"admin-job-type":0} /tidb/cdc/default/default/task/position/784eac33-2618-482f-8967-7501f5638dcc/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {UpstreamID:7365780572903040080 Namespace:default ID:9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:10:48.137501553 +0800 CST StartTs:449571572421754889 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00433b050 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571572447707139} {CheckpointTs:449571579643559943 MinTableBarrierTs:449571579643559943 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571579643559944, checkpointTs: 449571579643559943, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571579643559943, checkpointTs: 449571579643559943, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/784eac33-2618-482f-8967-7501f5638dcc {"id":"784eac33-2618-482f-8967-7501f5638dcc","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbd5c23ba 784eac33-2618-482f-8967-7501f5638dcc /tidb/cdc/default/default/changefeed/info/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"upstream-id":7365780572903040080,"namespace":"default","changefeed-id":"9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:10:48.137501553+08:00","start-ts":449571572421754889,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571572447707139} /tidb/cdc/default/default/changefeed/status/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":449571579643559943,"min-table-barrier-ts":449571579643559943,"admin-job-type":0} /tidb/cdc/default/default/task/position/784eac33-2618-482f-8967-7501f5638dcc/9dbdba3f-f6c4-4405-9acd-e7bf6643f6c9 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365780572903040080 {"id":7365780572903040080,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x }lease 22318f4cbd97ea1a revoked check diff failed 1-th time, retry later check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 14.70 secs (228852449 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 3-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:11:25 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32fa41480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:6535, start at 2024-05-06 15:11:48.34847101 +0800 CST m=+5.055726967 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:13:48.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:11:48.357 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:01:48.357 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32fa41480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:6535, start at 2024-05-06 15:11:48.34847101 +0800 CST m=+5.055726967 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:13:48.355 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:11:48.357 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:01:48.357 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d32fa42a80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:6617, start at 2024-05-06 15:11:48.399922072 +0800 CST m=+5.039698208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:13:48.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:11:48.394 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:01:48.394 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 27.15 secs (123918401 bytes/sec) [Pipeline] { [Pipeline] cache + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.7944.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449571589326372867 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571589326372867 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:11:53 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.79747976.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:11:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a91e6c9-eb50-4795-9e4d-cd0ec794e76f {"id":"4a91e6c9-eb50-4795-9e4d-cd0ec794e76f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbe66e500 4a91e6c9-eb50-4795-9e4d-cd0ec794e76f /tidb/cdc/default/default/upstream/7365780868233285055 {"id":7365780868233285055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a91e6c9-eb50-4795-9e4d-cd0ec794e76f {"id":"4a91e6c9-eb50-4795-9e4d-cd0ec794e76f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbe66e500 4a91e6c9-eb50-4795-9e4d-cd0ec794e76f /tidb/cdc/default/default/upstream/7365780868233285055 {"id":7365780868233285055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a91e6c9-eb50-4795-9e4d-cd0ec794e76f {"id":"4a91e6c9-eb50-4795-9e4d-cd0ec794e76f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cbe66e500 4a91e6c9-eb50-4795-9e4d-cd0ec794e76f /tidb/cdc/default/default/upstream/7365780868233285055 {"id":7365780868233285055,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8029.out cli changefeed create --start-ts=449571589326372867 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 1f2ea5c5-4108-494d-a0e9-14cf9627be77 Info: {"upstream_id":7365780868233285055,"namespace":"default","id":"1f2ea5c5-4108-494d-a0e9-14cf9627be77","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:11:57.421039903+08:00","start_ts":449571589326372867,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571589326372867,"checkpoint_ts":449571589326372867,"checkpoint_time":"2024-05-06 15:11:52.506"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "operationcount"="0" "mysql.user"="root" "workload"="core" "recordcount"="10" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.db"="testSync" "readproportion"="0" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 9.547496ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2176.3, Avg(us): 1833, Min(us): 927, Max(us): 4874, 95th(us): 5000, 99th(us): 5000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 9.02 secs (372975728 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 10.00 secs (336442740 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 16.72 secs (201258629 bytes/sec) [Pipeline] { [Pipeline] cache ***************** properties ***************** "recordcount"="10" "threadcount"="2" "readallfields"="true" "requestdistribution"="uniform" "updateproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "dotransactions"="false" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "readproportion"="0" "mysql.db"="testSync" "insertproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 3.850484ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3874.9, Avg(us): 683, Min(us): 421, Max(us): 1534, 95th(us): 2000, 99th(us): 2000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 17.50 secs (192314912 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 21.57 secs (155974367 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449571580469575715" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 15:13:20.513 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:13:20.514 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:13:20.514 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571579931918345] [resolvedTs=449571580469575715] [2024/05/06 15:13:20.515 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:13:20.515 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:13:20.516 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=2.354414ms] [2024/05/06 15:13:20.517 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=2.54566ms] [2024/05/06 15:13:20.519 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:13:20.522 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:13:20.522 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:13:20.524 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:13:20.524 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:13:20.527 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:13:20.529 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 15:13:20.529 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571579931918339,\"CommitTs\":449571579931918345,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":106,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:20.529 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571579931918339,\"CommitTs\":449571579931918345,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":106,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:20.737 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=207.573438ms] [namespace=] [changefeed=] [2024/05/06 15:13:20.737 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571579971239944,\"CommitTs\":449571579971239951,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:20.737 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571579971239944,\"CommitTs\":449571579971239951,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:20.877 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=140.107291ms] [namespace=] [changefeed=] [2024/05/06 15:13:20.877 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580023668747,\"CommitTs\":449571580023668750,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1_1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:21.078 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580023668747,\"CommitTs\":449571580023668750,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1_1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:21.217 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=139.002828ms] [namespace=] [changefeed=] [2024/05/06 15:13:21.217 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580063252492,\"CommitTs\":449571580063252497,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:21.418 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580063252492,\"CommitTs\":449571580063252497,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:21.494 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=76.550607ms] [namespace=] [changefeed=] [2024/05/06 15:13:21.494 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580089204751,\"CommitTs\":449571580089204755,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:21.494 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580089204751,\"CommitTs\":449571580089204755,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:21.568 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=73.586401ms] [namespace=] [changefeed=] [2024/05/06 15:13:21.568 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580115419152,\"CommitTs\":449571580115419155,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:21.568 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580115419152,\"CommitTs\":449571580115419155,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:21.643 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=75.175684ms] [namespace=] [changefeed=] [2024/05/06 15:13:21.643 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580194062348,\"CommitTs\":449571580207169537,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:21.844 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580194062348,\"CommitTs\":449571580207169537,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.107 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=262.911185ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.107 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580272705546,\"CommitTs\":449571580285812738,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.107 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580272705546,\"CommitTs\":449571580285812738,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.378 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=270.53179ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.378 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580338241547,\"CommitTs\":449571580338241550,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.378 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580338241547,\"CommitTs\":449571580338241550,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.580 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=201.810027ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.580 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580364455958,\"CommitTs\":449571580364455966,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.580 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580364455958,\"CommitTs\":449571580364455966,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.659 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=79.269196ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.659 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580403777546,\"CommitTs\":449571580416884737,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2_1\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.659 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580403777546,\"CommitTs\":449571580416884737,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2_1\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.803 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=143.874152ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.803 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580403777546,\"CommitTs\":449571580416884737,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.803 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580403777546,\"CommitTs\":449571580416884737,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:22.945 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=141.193471ms] [namespace=] [changefeed=] [2024/05/06 15:13:22.945 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580443361286,\"CommitTs\":449571580443361291,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:22.945 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580443361286,\"CommitTs\":449571580443361291,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:23.024 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=79.771548ms] [namespace=] [changefeed=] [2024/05/06 15:13:23.025 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571580469575709,\"CommitTs\":449571580469575715,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"check1\",\"tbl-id\":127,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:13:23.226 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571580469575709,\"CommitTs\":449571580469575715,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"check1\",\"tbl-id\":127,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:13:23.307 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=80.93341ms] [namespace=] [changefeed=] [2024/05/06 15:13:23.307 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.307 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.407 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.408 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.508 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.508 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449571580469575715] [2024/05/06 15:13:23.609 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449571580469575715] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 15:13:23.609 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Mon May 6 15:13:23 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCD4DFE155CE64 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 07:13:36 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 15.31 secs (219839618 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3301b9d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:4163, start at 2024-05-06 15:13:50.731243804 +0800 CST m=+5.158786023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:15:50.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:13:50.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:03:50.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3301b9d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:4163, start at 2024-05-06 15:13:50.731243804 +0800 CST m=+5.158786023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:15:50.738 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:13:50.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:03:50.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3301ba700010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:4244, start at 2024-05-06 15:13:50.764149395 +0800 CST m=+5.129265134 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:15:50.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:13:50.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:03:50.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 15:13:53 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.55635565.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:13:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 13.47 secs (249734636 bytes/sec) [Pipeline] { + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b7d2524-a7f6-4ff2-94b1-7547928956dd {"id":"5b7d2524-a7f6-4ff2-94b1-7547928956dd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427cf4 5b7d2524-a7f6-4ff2-94b1-7547928956dd /tidb/cdc/default/default/upstream/7365781378623468777 {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b7d2524-a7f6-4ff2-94b1-7547928956dd {"id":"5b7d2524-a7f6-4ff2-94b1-7547928956dd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427cf4 5b7d2524-a7f6-4ff2-94b1-7547928956dd /tidb/cdc/default/default/upstream/7365781378623468777 {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5b7d2524-a7f6-4ff2-94b1-7547928956dd {"id":"5b7d2524-a7f6-4ff2-94b1-7547928956dd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427cf4 5b7d2524-a7f6-4ff2-94b1-7547928956dd /tidb/cdc/default/default/upstream/7365781378623468777 {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=6dbdc7a1-d913-448c-b111-7fba3db2ac7e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-pl2f7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-pl2f7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=a03e3dba-8a2f-4199-9be7-6a6c1a03b0d3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-n19r9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-n19r9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=02d21d99-9a44-453f-8560-9cdc88641f45 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xpbw3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-xpbw3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: http_api http_api_tls api_v2 kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=53896944-aef3-40f1-84f3-75ac931e687c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-qqf95 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-qqf95 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=7f0f5332-8652-43f0-acce-b711283bfaa9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-wdwqq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-wdwqq ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [Pipeline] sh table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=7b6b9351-b02d-42bb-a283-9fb19b5d782b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-j01wm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-j01wm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=935f0aaf-38e2-442f-b540-7ed8435dd5fc BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ff23f GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ff23f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=57d4c574-bace-4efb-baf8-c3837ec00c58 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-x30bf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-x30bf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli savepoint synced_status PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=06f585ef-bcd1-4db9-b076-c62a8f1d6d01 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-ncld8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) table consistent_replicate_gbk.check1 not exists for 2-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=a2bb0b27-b6af-4c4e-84b0-313fef83088a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-88wx8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-88wx8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="10" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "workload"="core" "threadcount"="2" "readallfields"="true" "insertproportion"="0" "mysql.db"="testSync" "updateproportion"="0" "mysql.port"="4000" "scanproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.115615ms INSERT - Takes(s): 0.0, Count: 10, OPS: 4115.4, Avg(us): 768, Min(us): 478, Max(us): 1711, 95th(us): 2000, 99th(us): 2000 Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Installing collected packages: urllib3, idna, certifi, requests Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 The 1 times to try to start tidb cluster... table consistent_replicate_gbk.check1 exists check diff successfully Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=d7d100e4-c146-42cd-a3b4-224adfd68cee BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-jktf9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-jktf9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:14:06 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.57715773.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:09 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/299c279c-a16c-4abb-a052-9ac0ad7b31bb {UpstreamID:7365781378623468777 Namespace:default ID:299c279c-a16c-4abb-a052-9ac0ad7b31bb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:13:58.202400906 +0800 CST StartTs:449571622252969986 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f1e6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571622265815048} {CheckpointTs:449571622344720409 MinTableBarrierTs:449571625018327043 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb61c53d-56f7-4b45-8217-95ede43af0c3 {"id":"cb61c53d-56f7-4b45-8217-95ede43af0c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427d71 cb61c53d-56f7-4b45-8217-95ede43af0c3 /tidb/cdc/default/default/changefeed/info/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"upstream-id":7365781378623468777,"namespace":"default","changefeed-id":"299c279c-a16c-4abb-a052-9ac0ad7b31bb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:13:58.202400906+08:00","start-ts":449571622252969986,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571622265815048} /tidb/cdc/default/default/changefeed/status/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":449571622344720409,"min-table-barrier-ts":449571625018327043,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb61c53d-56f7-4b45-8217-95ede43af0c3/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781378623468777 {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/299c279c-a16c-4abb-a052-9ac0ad7b31bb {UpstreamID:7365781378623468777 Namespace:default ID:299c279c-a16c-4abb-a052-9ac0ad7b31bb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:13:58.202400906 +0800 CST StartTs:449571622252969986 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f1e6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571622265815048} {CheckpointTs:449571622344720409 MinTableBarrierTs:449571625018327043 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb61c53d-56f7-4b45-8217-95ede43af0c3 {"id":"cb61c53d-56f7-4b45-8217-95ede43af0c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427d71 cb61c53d-56f7-4b45-8217-95ede43af0c3 /tidb/cdc/default/default/changefeed/info/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"upstream-id":7365781378623468777,"namespace":"default","changefeed-id":"299c279c-a16c-4abb-a052-9ac0ad7b31bb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:13:58.202400906+08:00","start-ts":449571622252969986,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571622265815048} /tidb/cdc/default/default/changefeed/status/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":449571622344720409,"min-table-barrier-ts":449571625018327043,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb61c53d-56f7-4b45-8217-95ede43af0c3/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781378623468777 {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/299c279c-a16c-4abb-a052-9ac0ad7b31bb {UpstreamID:7365781378623468777 Namespace:default ID:299c279c-a16c-4abb-a052-9ac0ad7b31bb SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:13:58.202400906 +0800 CST StartTs:449571622252969986 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f1e6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571622265815048} {CheckpointTs:449571622344720409 MinTableBarrierTs:449571625018327043 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571622344720409, checkpointTs: 449571622344720409, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb61c53d-56f7-4b45-8217-95ede43af0c3 {"id":"cb61c53d-56f7-4b45-8217-95ede43af0c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0427d71 cb61c53d-56f7-4b45-8217-95ede43af0c3 /tidb/cdc/default/default/changefeed/info/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"upstream-id":7365781378623468777,"namespace":"default","changefeed-id":"299c279c-a16c-4abb-a052-9ac0ad7b31bb","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:13:58.202400906+08:00","start-ts":449571622252969986,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571622265815048} /tidb/cdc/default/default/changefeed/status/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":449571622344720409,"min-table-barrier-ts":449571625018327043,"admin-job-type":0} /tidb/cdc/default/default/task/position/cb61c53d-56f7-4b45-8217-95ede43af0c3/299c279c-a16c-4abb-a052-9ac0ad7b31bb {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781378623468777 + grep -q 'etcd info' {"id":7365781378623468777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33030b8c001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:1188, start at 2024-05-06 15:14:12.371046033 +0800 CST m=+5.260839067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:12.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:12.373 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:12.373 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33030b8c001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:1188, start at 2024-05-06 15:14:12.371046033 +0800 CST m=+5.260839067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:12.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:12.373 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:12.373 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33030c400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:1271, start at 2024-05-06 15:14:12.402921779 +0800 CST m=+5.226506883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:12.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:12.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:12.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 299c279c-a16c-4abb-a052-9ac0ad7b31bb 449571626564976642 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/299c279c-a16c-4abb-a052-9ac0ad7b31bb/meta global resolved ts 449571625202090011 not forward to 449571626564976642 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330330340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:1184, start at 2024-05-06 15:14:14.715825803 +0800 CST m=+5.312268260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:14.723 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:14.719 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:14.719 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:14:15 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.25902592.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33031a280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:1185, start at 2024-05-06 15:14:13.30017881 +0800 CST m=+5.350207507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:13.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:13.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:13.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33031a280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:1185, start at 2024-05-06 15:14:13.30017881 +0800 CST m=+5.350207507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:13.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:13.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:13.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33031a340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:1257, start at 2024-05-06 15:14:13.29079695 +0800 CST m=+5.285148348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:13.298 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:13.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:13.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_redo_resolved_ts 299c279c-a16c-4abb-a052-9ac0ad7b31bb 449571626564976642 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/299c279c-a16c-4abb-a052-9ac0ad7b31bb/meta global resolved ts 449571625202090011 not forward to 449571626564976642 run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330330340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:1184, start at 2024-05-06 15:14:14.715825803 +0800 CST m=+5.312268260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:14.723 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:14.719 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:14.719 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330330f00003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:1273, start at 2024-05-06 15:14:14.718533885 +0800 CST m=+5.263626536 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:14.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:14.716 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:14.716 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2550.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330359b80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:1253, start at 2024-05-06 15:14:17.374454154 +0800 CST m=+5.300783025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.375 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.375 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330362fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:1192, start at 2024-05-06 15:14:17.921021028 +0800 CST m=+5.209429879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330362fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:1192, start at 2024-05-06 15:14:17.921021028 +0800 CST m=+5.209429879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330363c0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:1280, start at 2024-05-06 15:14:17.981953551 +0800 CST m=+5.218312264 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330364080002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:1303, start at 2024-05-06 15:14:17.987051302 +0800 CST m=+5.306930858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.986 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.986 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3303411c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:1251, start at 2024-05-06 15:14:15.784844684 +0800 CST m=+5.241492796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:15.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:15.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:15.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3303411c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:1251, start at 2024-05-06 15:14:15.784844684 +0800 CST m=+5.241492796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:15.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:15.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:15.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33034224000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:1338, start at 2024-05-06 15:14:15.827733775 +0800 CST m=+5.235779969 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:15.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:15.817 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:15.817 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f {"id":"ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a07eef ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f /tidb/cdc/default/default/upstream/7365781488293430623 {"id":7365781488293430623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f {"id":"ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a07eef ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f /tidb/cdc/default/default/upstream/7365781488293430623 {"id":7365781488293430623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f {"id":"ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a07eef ec571ebc-01ba-4b3e-ac5b-5bb0b332b01f /tidb/cdc/default/default/upstream/7365781488293430623 {"id":7365781488293430623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 6f8e540c-51ac-4dbb-875f-8f8f7c596172 Info: {"upstream_id":7365781488293430623,"namespace":"default","id":"6f8e540c-51ac-4dbb-875f-8f8f7c596172","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:14:18.881307805+08:00","start_ts":449571626817421313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571626817421313,"checkpoint_ts":449571626817421313,"checkpoint_time":"2024-05-06 15:14:15.523"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330362700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:1134, start at 2024-05-06 15:14:17.923745928 +0800 CST m=+5.095702848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330362700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:1134, start at 2024-05-06 15:14:17.923745928 +0800 CST m=+5.095702848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.934 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.934 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 8.66 secs (388766376 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330366740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:1193, start at 2024-05-06 15:14:18.177172409 +0800 CST m=+5.306844612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:18.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:18.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:18.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2572.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449571627613028353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571627613028353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:14:20 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.25762578.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330359b80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:1253, start at 2024-05-06 15:14:17.374454154 +0800 CST m=+5.300783025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.375 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.375 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33035b380015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:1330, start at 2024-05-06 15:14:17.466795882 +0800 CST m=+5.326933205 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.473 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.473 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037cc40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:1327, start at 2024-05-06 15:14:19.609262081 +0800 CST m=+5.369042205 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.619 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.619 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330364080002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:1303, start at 2024-05-06 15:14:17.987051302 +0800 CST m=+5.306930858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:17.995 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:17.986 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:17.986 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330363e40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:1384, start at 2024-05-06 15:14:18.020913545 +0800 CST m=+5.281865038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:18.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:18.028 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:18.028 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 15:14:20 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.25932595.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037f580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:1230, start at 2024-05-06 15:14:19.756663121 +0800 CST m=+6.873108878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_redo_resolved_ts 299c279c-a16c-4abb-a052-9ac0ad7b31bb 449571626564976642 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/299c279c-a16c-4abb-a052-9ac0ad7b31bb/meta run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330366740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:1193, start at 2024-05-06 15:14:18.177172409 +0800 CST m=+5.306844612 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:18.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:18.141 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:18.141 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330367f80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:1271, start at 2024-05-06 15:14:18.241350004 +0800 CST m=+5.296649148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:18.248 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:18.238 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:18.238 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449571627983175681 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571627983175681 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2600.out cli tso query --pd=http://127.0.0.1:2379 ***************** properties ***************** "workload"="core" "requestdistribution"="uniform" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.db"="sink_retry" "recordcount"="10" "mysql.user"="root" "threadcount"="2" "scanproportion"="0" "mysql.port"="4000" "readallfields"="true" "readproportion"="0" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 24.337777ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1640.0, Avg(us): 4756, Min(us): 1176, Max(us): 18121, 95th(us): 19000, 99th(us): 19000 [Mon May 6 15:14:21 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26342636.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/05/06 15:14:22.430 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:14:22.432 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:14:22.435 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:14:22.435 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571625202090011] [resolvedTs=449571627377623045] [2024/05/06 15:14:22.436 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:14:22.436 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:14:22.436 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:14:22.436 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:14:22.439 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.361729ms] [2024/05/06 15:14:22.440 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=5.497608ms] [2024/05/06 15:14:22.441 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:14:22.443 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:14:22.444 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:14:22.443 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:14:22.445 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:14:22.445 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:14:22.449 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:14:22.450 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 15:14:22.450 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449571625202090006,\"CommitTs\":449571625202090011,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:14:22.451 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449571625202090006,\"CommitTs\":449571625202090011,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 15:14:22.452 +08:00] [INFO] [mysql_ddl_sink.go:119] ["Execute DDL failed, but error can be ignored"] [startTs=449571625202090006] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037a04000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:1314, start at 2024-05-06 15:14:19.412252208 +0800 CST m=+5.227234092 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.393 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.393 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037a04000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:1314, start at 2024-05-06 15:14:19.412252208 +0800 CST m=+5.227234092 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.393 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.393 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037b600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:1386, start at 2024-05-06 15:14:19.49798984 +0800 CST m=+5.260578193 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.480 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.480 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037cc40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:1327, start at 2024-05-06 15:14:19.609262081 +0800 CST m=+5.369042205 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.619 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.619 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33037cac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:1408, start at 2024-05-06 15:14:19.598357347 +0800 CST m=+5.293874767 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:19.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:19.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:19.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:14:22.652 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449571627377623045] [2024/05/06 15:14:22.652 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449571627377623045] [2024/05/06 15:14:22.753 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449571627377623045] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:14:22.753 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully check diff successfully [Mon May 6 15:14:22 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT + set +x + tso='449571628337070081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571628337070081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:14:22 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26352637.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2630.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:23 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bf3314f-cd0d-4d26-a08d-00a0c25ea913 {"id":"9bf3314f-cd0d-4d26-a08d-00a0c25ea913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc09be4f7 9bf3314f-cd0d-4d26-a08d-00a0c25ea913 /tidb/cdc/default/default/upstream/7365781485978103379 {"id":7365781485978103379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bf3314f-cd0d-4d26-a08d-00a0c25ea913 {"id":"9bf3314f-cd0d-4d26-a08d-00a0c25ea913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc09be4f7 9bf3314f-cd0d-4d26-a08d-00a0c25ea913 /tidb/cdc/default/default/upstream/7365781485978103379 {"id":7365781485978103379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9bf3314f-cd0d-4d26-a08d-00a0c25ea913 {"id":"9bf3314f-cd0d-4d26-a08d-00a0c25ea913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc09be4f7 9bf3314f-cd0d-4d26-a08d-00a0c25ea913 /tidb/cdc/default/default/upstream/7365781485978103379 {"id":7365781485978103379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2622.out cli changefeed create --start-ts=449571627613028353 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2654.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 15:14:23 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26072609.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 94f10709-5347-4dc6-b904-e74b9e8e9784 Info: {"upstream_id":7365781485978103379,"namespace":"default","id":"94f10709-5347-4dc6-b904-e74b9e8e9784","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:14:23.555014722+08:00","start_ts":449571627613028353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571627613028353,"checkpoint_ts":449571627613028353,"checkpoint_time":"2024-05-06 15:14:18.558"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:23 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 {"id":"eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a806f3 eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 /tidb/cdc/default/default/upstream/7365781500205727194 {"id":7365781500205727194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 {"id":"eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a806f3 eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 /tidb/cdc/default/default/upstream/7365781500205727194 {"id":7365781500205727194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 {"id":"eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a806f3 eac2eac5-7bcf-4c1a-bee7-fd00b5665fc2 /tidb/cdc/default/default/upstream/7365781500205727194 {"id":7365781500205727194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 + set +x + tso='449571628692537345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571628692537345 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:14:24 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.26592661.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 15:14:24 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.26602662.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a56da47-f6d3-440b-8b5e-0ef5beed240e {"id":"7a56da47-f6d3-440b-8b5e-0ef5beed240e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a1a609 7a56da47-f6d3-440b-8b5e-0ef5beed240e /tidb/cdc/default/default/upstream/7365781495280878086 {"id":7365781495280878086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a56da47-f6d3-440b-8b5e-0ef5beed240e {"id":"7a56da47-f6d3-440b-8b5e-0ef5beed240e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a1a609 7a56da47-f6d3-440b-8b5e-0ef5beed240e /tidb/cdc/default/default/upstream/7365781495280878086 {"id":7365781495280878086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a56da47-f6d3-440b-8b5e-0ef5beed240e {"id":"7a56da47-f6d3-440b-8b5e-0ef5beed240e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0a1a609 7a56da47-f6d3-440b-8b5e-0ef5beed240e /tidb/cdc/default/default/upstream/7365781495280878086 {"id":7365781495280878086,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2686.out cli changefeed create --start-ts=449571627983175681 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' + set +x + tso='449571628865552385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571628865552385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:14:24 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.26852687.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2653.out cli tso query --pd=http://127.0.0.1:2379 + set +x Create changefeed successfully! ID: b0e44d27-eef5-4913-b5f9-10d9559ae8ee Info: {"upstream_id":7365781495280878086,"namespace":"default","id":"b0e44d27-eef5-4913-b5f9-10d9559ae8ee","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:14:25.093536402+08:00","start_ts":449571627983175681,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571627983175681,"checkpoint_ts":449571627983175681,"checkpoint_time":"2024-05-06 15:14:19.970"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Mon May 6 15:14:25 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27272729.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:25 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4552d710-2ed7-45bd-9555-af63ee0d772e {"id":"4552d710-2ed7-45bd-9555-af63ee0d772e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ac95f2 4552d710-2ed7-45bd-9555-af63ee0d772e /tidb/cdc/default/default/upstream/7365781509801156003 {"id":7365781509801156003,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4552d710-2ed7-45bd-9555-af63ee0d772e {"id":"4552d710-2ed7-45bd-9555-af63ee0d772e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ac95f2 4552d710-2ed7-45bd-9555-af63ee0d772e /tidb/cdc/default/default/upstream/7365781509801156003 {"id":7365781509801156003,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4552d710-2ed7-45bd-9555-af63ee0d772e {"id":"4552d710-2ed7-45bd-9555-af63ee0d772e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ac95f2 4552d710-2ed7-45bd-9555-af63ee0d772e /tidb/cdc/default/default/upstream/7365781509801156003 {"id":7365781509801156003,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2681.out cli changefeed create --start-ts=449571628337070081 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Create changefeed successfully! ID: cd47b60b-e5a6-44d5-8bbc-a70fec576349 Info: {"upstream_id":7365781509801156003,"namespace":"default","id":"cd47b60b-e5a6-44d5-8bbc-a70fec576349","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:14:26.200127973+08:00","start_ts":449571628337070081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571628337070081,"checkpoint_ts":449571628337070081,"checkpoint_time":"2024-05-06 15:14:21.320"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449571629247758337 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571629247758337 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:14:26 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.26922694.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 {"id":"3f3f6c9f-da6b-48c0-b40f-fae64cc185e4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b4d9f2 3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 /tidb/cdc/default/default/upstream/7365781507830678226 {"id":7365781507830678226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 {"id":"3f3f6c9f-da6b-48c0-b40f-fae64cc185e4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b4d9f2 3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 /tidb/cdc/default/default/upstream/7365781507830678226 {"id":7365781507830678226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 {"id":"3f3f6c9f-da6b-48c0-b40f-fae64cc185e4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b4d9f2 3f3f6c9f-da6b-48c0-b40f-fae64cc185e4 /tidb/cdc/default/default/upstream/7365781507830678226 {"id":7365781507830678226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff71af4c-86b4-4bee-a234-d39865b8fd8c {"id":"ff71af4c-86b4-4bee-a234-d39865b8fd8c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0afecf5 ff71af4c-86b4-4bee-a234-d39865b8fd8c /tidb/cdc/default/default/upstream/7365781504529130118 {"id":7365781504529130118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff71af4c-86b4-4bee-a234-d39865b8fd8c {"id":"ff71af4c-86b4-4bee-a234-d39865b8fd8c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0afecf5 ff71af4c-86b4-4bee-a234-d39865b8fd8c /tidb/cdc/default/default/upstream/7365781504529130118 {"id":7365781504529130118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ff71af4c-86b4-4bee-a234-d39865b8fd8c {"id":"ff71af4c-86b4-4bee-a234-d39865b8fd8c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0afecf5 ff71af4c-86b4-4bee-a234-d39865b8fd8c /tidb/cdc/default/default/upstream/7365781504529130118 {"id":7365781504529130118,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2705.out cli changefeed create --start-ts=449571628692537345 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic Create changefeed successfully! ID: 3d280be8-9d01-46ac-be16-4a931b0b9897 Info: {"upstream_id":7365781504529130118,"namespace":"default","id":"3d280be8-9d01-46ac-be16-4a931b0b9897","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-06T15:14:27.630710198+08:00","start_ts":449571628692537345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571628692537345,"checkpoint_ts":449571628692537345,"checkpoint_time":"2024-05-06 15:14:22.676"} PASS + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:27 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76df06cb-02c2-4966-87ba-844801b9e7ab {"id":"76df06cb-02c2-4966-87ba-844801b9e7ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b05df4 76df06cb-02c2-4966-87ba-844801b9e7ab /tidb/cdc/default/default/upstream/7365781509519205024 {"id":7365781509519205024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76df06cb-02c2-4966-87ba-844801b9e7ab {"id":"76df06cb-02c2-4966-87ba-844801b9e7ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b05df4 76df06cb-02c2-4966-87ba-844801b9e7ab /tidb/cdc/default/default/upstream/7365781509519205024 {"id":7365781509519205024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76df06cb-02c2-4966-87ba-844801b9e7ab {"id":"76df06cb-02c2-4966-87ba-844801b9e7ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b05df4 76df06cb-02c2-4966-87ba-844801b9e7ab /tidb/cdc/default/default/upstream/7365781509519205024 {"id":7365781509519205024,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2738.out cli changefeed create --start-ts=449571628865552385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: 89b39ca5-9ad3-461c-9910-7474c3e7fb10 Info: {"upstream_id":7365781509519205024,"namespace":"default","id":"89b39ca5-9ad3-461c-9910-7474c3e7fb10","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:14:28.294079892+08:00","start_ts":449571628865552385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571628865552385,"checkpoint_ts":449571628865552385,"checkpoint_time":"2024-05-06 15:14:23.336"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37667311-0411-4607-ba5c-7d93697b8407 {"id":"37667311-0411-4607-ba5c-7d93697b8407","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0af0265 37667311-0411-4607-ba5c-7d93697b8407 /tidb/cdc/default/default/upstream/7365781498331835758 {"id":7365781498331835758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37667311-0411-4607-ba5c-7d93697b8407 {"id":"37667311-0411-4607-ba5c-7d93697b8407","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0af0265 37667311-0411-4607-ba5c-7d93697b8407 /tidb/cdc/default/default/upstream/7365781498331835758 {"id":7365781498331835758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37667311-0411-4607-ba5c-7d93697b8407 {"id":"37667311-0411-4607-ba5c-7d93697b8407","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0af0265 37667311-0411-4607-ba5c-7d93697b8407 /tidb/cdc/default/default/upstream/7365781498331835758 {"id":7365781498331835758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 c6c9fb5c-d420-4396-8aa8-9c12141db291 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=c6c9fb5c-d420-4396-8aa8-9c12141db291 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c6c9fb5c-d420-4396-8aa8-9c12141db291 -s + info='{ "upstream_id": 7365781498331835758, "namespace": "default", "id": "c6c9fb5c-d420-4396-8aa8-9c12141db291", "state": "normal", "checkpoint_tso": 449571630230011908, "checkpoint_time": "2024-05-06 15:14:28.541", "error": null }' + echo '{ "upstream_id": 7365781498331835758, "namespace": "default", "id": "c6c9fb5c-d420-4396-8aa8-9c12141db291", "state": "normal", "checkpoint_tso": 449571630230011908, "checkpoint_time": "2024-05-06 15:14:28.541", "error": null }' { "upstream_id": 7365781498331835758, "namespace": "default", "id": "c6c9fb5c-d420-4396-8aa8-9c12141db291", "state": "normal", "checkpoint_tso": 449571630230011908, "checkpoint_time": "2024-05-06 15:14:28.541", "error": null } ++ echo '{' '"upstream_id":' 7365781498331835758, '"namespace":' '"default",' '"id":' '"c6c9fb5c-d420-4396-8aa8-9c12141db291",' '"state":' '"normal",' '"checkpoint_tso":' 449571630230011908, '"checkpoint_time":' '"2024-05-06' '15:14:28.541",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781498331835758, '"namespace":' '"default",' '"id":' '"c6c9fb5c-d420-4396-8aa8-9c12141db291",' '"state":' '"normal",' '"checkpoint_tso":' 449571630230011908, '"checkpoint_time":' '"2024-05-06' '15:14:28.541",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully "is-owner": true, run task successfully + set +x go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/prometheus/common v0.53.0 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a76253ed-35ce-47ff-9ef5-c9f01e7625e1 {"id":"a76253ed-35ce-47ff-9ef5-c9f01e7625e1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b255f3 a76253ed-35ce-47ff-9ef5-c9f01e7625e1 /tidb/cdc/default/default/upstream/7365781515679440346 {"id":7365781515679440346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a76253ed-35ce-47ff-9ef5-c9f01e7625e1 {"id":"a76253ed-35ce-47ff-9ef5-c9f01e7625e1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b255f3 a76253ed-35ce-47ff-9ef5-c9f01e7625e1 /tidb/cdc/default/default/upstream/7365781515679440346 {"id":7365781515679440346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a76253ed-35ce-47ff-9ef5-c9f01e7625e1 {"id":"a76253ed-35ce-47ff-9ef5-c9f01e7625e1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0b255f3 a76253ed-35ce-47ff-9ef5-c9f01e7625e1 /tidb/cdc/default/default/upstream/7365781515679440346 {"id":7365781515679440346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2737.out cli changefeed create --start-ts=449571629247758337 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 1a8e29ca-b77d-45ba-be0a-889a088e04c1 Info: {"upstream_id":7365781515679440346,"namespace":"default","id":"1a8e29ca-b77d-45ba-be0a-889a088e04c1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:14:29.695116823+08:00","start_ts":449571629247758337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571629247758337,"checkpoint_ts":449571629247758337,"checkpoint_time":"2024-05-06 15:14:24.794"} PASS go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 coverage: 2.4% of statements in github.com/pingcap/tiflow/... go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365781500205727194 449571630246002691 449571629053247492 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 owner pid: 2665 owner id c9518583-77fc-4cae-9e67-3aca5cbc34cd default [Mon May 6 15:14:30 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27812783.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/text v0.14.0 run task successfully table test.finish_mark not exists for 2-th check, retry later check_changefeed_state http://127.0.0.1:2379 d44ce6d3-80cb-483f-932a-190c57632daf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=d44ce6d3-80cb-483f-932a-190c57632daf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d44ce6d3-80cb-483f-932a-190c57632daf -s + info='{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' + echo '{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' { "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null } ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"stopped",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"stopped",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365781500205727194 + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/tools v0.20.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 check diff failed 2-th time, retry later go: downloading golang.org/x/text v0.14.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 table test.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:33 GMT < Content-Length: 911 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/c9518583-77fc-4cae-9e67-3aca5cbc34cd {"id":"c9518583-77fc-4cae-9e67-3aca5cbc34cd","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba4304 c9518583-77fc-4cae-9e67-3aca5cbc34cd /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "c9518583-77fc-4cae-9e67-3aca5cbc34cd default" | grep id check diff successfully go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 run task successfully check_changefeed_state http://127.0.0.1:2379 d44ce6d3-80cb-483f-932a-190c57632daf normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=d44ce6d3-80cb-483f-932a-190c57632daf + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d44ce6d3-80cb-483f-932a-190c57632daf -s wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 4-th check, retry later go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 + info='{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "normal", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' + echo '{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "normal", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' { "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "normal", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null } ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"normal",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"normal",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365781500205727194 449571630507884550 449571630507884551 wait process cdc.test exit for 2-th time... go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible table foreign_key.finish_mark not exists for 1-th check, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 "id": "1f2ac58f-68c2-4756-90df-05963229d0d8", run task successfully wait process cdc.test exit for 3-th time... table foreign_key.finish_mark not exists for 2-th check, retry later go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 13.53 secs (248733281 bytes/sec) [Pipeline] { [Pipeline] cache cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:14:35 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> run task successfully check_changefeed_state http://127.0.0.1:2379 d44ce6d3-80cb-483f-932a-190c57632daf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=d44ce6d3-80cb-483f-932a-190c57632daf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d44ce6d3-80cb-483f-932a-190c57632daf -s + info='{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' + echo '{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null }' { "upstream_id": 7365781500205727194, "namespace": "default", "id": "d44ce6d3-80cb-483f-932a-190c57632daf", "state": "stopped", "checkpoint_tso": 449571630507884551, "checkpoint_time": "2024-05-06 15:14:29.601", "error": null } ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"stopped",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"d44ce6d3-80cb-483f-932a-190c57632daf",' '"state":' '"stopped",' '"checkpoint_tso":' 449571630507884551, '"checkpoint_time":' '"2024-05-06' '15:14:29.601",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 0bb8a1ed-21a2-4041-851d-91467fc4388d normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=0bb8a1ed-21a2-4041-851d-91467fc4388d + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0bb8a1ed-21a2-4041-851d-91467fc4388d -s table test.finish_mark exists check table exists success check diff successfully + info='{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "0bb8a1ed-21a2-4041-851d-91467fc4388d", "state": "normal", "checkpoint_tso": 449571632251142146, "checkpoint_time": "2024-05-06 15:14:36.251", "error": null }' + echo '{ "upstream_id": 7365781500205727194, "namespace": "default", "id": "0bb8a1ed-21a2-4041-851d-91467fc4388d", "state": "normal", "checkpoint_tso": 449571632251142146, "checkpoint_time": "2024-05-06 15:14:36.251", "error": null }' { "upstream_id": 7365781500205727194, "namespace": "default", "id": "0bb8a1ed-21a2-4041-851d-91467fc4388d", "state": "normal", "checkpoint_tso": 449571632251142146, "checkpoint_time": "2024-05-06 15:14:36.251", "error": null } ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"0bb8a1ed-21a2-4041-851d-91467fc4388d",' '"state":' '"normal",' '"checkpoint_tso":' 449571632251142146, '"checkpoint_time":' '"2024-05-06' '15:14:36.251",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781500205727194, '"namespace":' '"default",' '"id":' '"0bb8a1ed-21a2-4041-851d-91467fc4388d",' '"state":' '"normal",' '"checkpoint_tso":' 449571632251142146, '"checkpoint_time":' '"2024-05-06' '15:14:36.251",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365781500205727194 capture_id: 1f2ac58f-68c2-4756-90df-05963229d0d8 pass test: check health pass test: get status start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... pass test: create changefeed go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:14:37 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table row_format.finish_mark not exists for 2-th check, retry later table test.simple1 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later run task successfully go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Changefeed remove successfully. ID: d44ce6d3-80cb-483f-932a-190c57632daf CheckpointTs: 449571630507884551 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7365781500205727194 449571633129586692 449571632251142146 449571630507884551 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release run task successfully table row_format.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later Changefeed remove successfully. ID: 0bb8a1ed-21a2-4041-851d-91467fc4388d CheckpointTs: 449571633391730694 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7365781500205727194 run task successfully Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.simple1 not exists for 2-th check, retry later Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:14:43 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> table row_format.finish_mark not exists for 4-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 5.75 secs (584853300 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 6-th check, retry later table test.simple1 exists pass test: list changefeed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: get changefeed pass test: pause changefeed table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later pass test: update changefeed wait process 2665 exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 2665 exit for 2-th time... table row_format.finish_mark not exists for 6-th check, retry later wait process 2665 exit for 3-th time... process 2665 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 1f2ac58f-68c2-4756-90df-05963229d0d8 -A1 | grep '"is-owner": true' table foreign_key.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33052e700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:7005, start at 2024-05-06 15:14:47.330323803 +0800 CST m=+5.286120737 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:47.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:47.324 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:47.324 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully [Mon May 6 15:14:48 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.30163018.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 7-th check, retry later table foreign_key.finish_mark exists check diff successfully table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33052e700006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:7005, start at 2024-05-06 15:14:47.330323803 +0800 CST m=+5.286120737 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:47.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:47.324 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:47.324 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33052fdc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:7092, start at 2024-05-06 15:14:47.437159031 +0800 CST m=+5.317877037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:16:47.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:14:47.415 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:04:47.415 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 5.59 secs (602130661 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:14:51 CST 2024] <<<<<< run test case foreign_key success! >>>>>> table row_format.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:51 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:14:36.868417545 +0800 CST StartTs:449571632386932748 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002de8bd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632400039948} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571636201127945, checkpointTs: 449571636201127942, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 15:14:37.085622042 +0800 CST StartTs:449571632452468750 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039330e0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632465575941} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/da775a47-4324-4d8f-8534-8a76a2f1c929 {"id":"da775a47-4324-4d8f-8534-8a76a2f1c929","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba43a8 da775a47-4324-4d8f-8534-8a76a2f1c929 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:14:36.868417545+08:00","start-ts":449571632386932748,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632400039948} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:36.98524993+08:00","start-ts":449571632426254348,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571634536775685} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:37.085622042+08:00","start-ts":449571632452468750,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632465575941} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571634366382085,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:14:36.868417545 +0800 CST StartTs:449571632386932748 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002de8bd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632400039948} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571636201127945, checkpointTs: 449571636201127942, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 15:14:37.085622042 +0800 CST StartTs:449571632452468750 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039330e0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632465575941} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/da775a47-4324-4d8f-8534-8a76a2f1c929 {"id":"da775a47-4324-4d8f-8534-8a76a2f1c929","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba43a8 da775a47-4324-4d8f-8534-8a76a2f1c929 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:14:36.868417545+08:00","start-ts":449571632386932748,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632400039948} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:36.98524993+08:00","start-ts":449571632426254348,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571634536775685} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:37.085622042+08:00","start-ts":449571632452468750,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632465575941} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571634366382085,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:14:36.868417545 +0800 CST StartTs:449571632386932748 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002de8bd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632400039948} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571636201127945, checkpointTs: 449571636201127942, state: Replicating changefeedID: default/changefeed-test3 {UpstreamID:7365781516446986661 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 15:14:37.085622042 +0800 CST StartTs:449571632452468750 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0039330e0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571632465575941} {CheckpointTs:449571632596647951 MinTableBarrierTs:449571636201127942 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571632596647951, checkpointTs: 449571632596647951, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f2ac58f-68c2-4756-90df-05963229d0d8 {"id":"1f2ac58f-68c2-4756-90df-05963229d0d8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/da775a47-4324-4d8f-8534-8a76a2f1c929 {"id":"da775a47-4324-4d8f-8534-8a76a2f1c929","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba432c 1f2ac58f-68c2-4756-90df-05963229d0d8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc0ba43a8 da775a47-4324-4d8f-8534-8a76a2f1c929 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:14:36.868417545+08:00","start-ts":449571632386932748,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632400039948} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:36.98524993+08:00","start-ts":449571632426254348,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571634536775685} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365781516446986661,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T15:14:37.085622042+08:00","start-ts":449571632452468750,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571632465575941} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571634366382085,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449571632596647951,"min-table-barrier-ts":449571636201127942,"admin-job-type":0} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1f2ac58f-68c2-4756-90df-05963229d0d8/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/da775a47-4324-4d8f-8534-8a76a2f1c929/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781516446986661 {"id":7365781516446986661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Mon May 6 15:14:52 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.83498351.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "address": "127.0.0.1:8300", run task successfully table row_format.finish_mark not exists for 9-th check, retry later pass test: list changefeed Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release pass test: get changefeed pass test: resume changefeed pass test: pause changefeed pass test: rebalance table table row_format.finish_mark not exists for 10-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... pass test: move table Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:14:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b830115-2011-4094-88bf-5ba55a88ef69 {"id":"2b830115-2011-4094-88bf-5ba55a88ef69","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc12165f5 2b830115-2011-4094-88bf-5ba55a88ef69 /tidb/cdc/default/default/upstream/7365781633630419264 {"id":7365781633630419264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b830115-2011-4094-88bf-5ba55a88ef69 {"id":"2b830115-2011-4094-88bf-5ba55a88ef69","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc12165f5 2b830115-2011-4094-88bf-5ba55a88ef69 /tidb/cdc/default/default/upstream/7365781633630419264 {"id":7365781633630419264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b830115-2011-4094-88bf-5ba55a88ef69 {"id":"2b830115-2011-4094-88bf-5ba55a88ef69","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc12165f5 2b830115-2011-4094-88bf-5ba55a88ef69 /tidb/cdc/default/default/upstream/7365781633630419264 {"id":7365781633630419264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "insertproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "mysql.port"="4000" "dotransactions"="false" "readallfields"="true" "mysql.db"="consistent_replicate_nfs" "scanproportion"="0" "fieldcount"="3" "threadcount"="10" "updateproportion"="0" "workload"="core" "recordcount"="50000" "mysql.user"="root" "readproportion"="0" "operationcount"="0" ********************************************** Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... pass test: get processors pass test: list processors ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully pass test: set log level table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "scanproportion"="0" "mysql.port"="4000" "mysql.db"="sink_retry" "readallfields"="true" "insertproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "threadcount"="2" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "operationcount"="0" "mysql.user"="root" "recordcount"="10" ********************************************** Run finished, takes 4.590054ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3135.2, Avg(us): 817, Min(us): 549, Max(us): 1876, 95th(us): 2000, 99th(us): 2000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:14:58 CST 2024] <<<<<< run test case row_format success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 1-th time... pass test: remove changefeed pass test: resign owner wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:15:00 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330605500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:3676, start at 2024-05-06 15:15:01.120503331 +0800 CST m=+5.225978170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:01.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:01.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:01.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330605500019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:3676, start at 2024-05-06 15:15:01.120503331 +0800 CST m=+5.225978170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:01.127 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:01.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:01.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330606c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:3758, start at 2024-05-06 15:15:01.190443496 +0800 CST m=+5.226723541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:01.197 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:01.170 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:01.170 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330620140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:4003, start at 2024-05-06 15:15:02.807823777 +0800 CST m=+5.124414852 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:02.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:02.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:02.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330620140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:4003, start at 2024-05-06 15:15:02.807823777 +0800 CST m=+5.124414852 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:02.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:02.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:02.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3306217c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:4080, start at 2024-05-06 15:15:02.908319974 +0800 CST m=+5.167494392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:02.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:02.879 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:02.879 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:15:06 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.53365338.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 INSERT - Takes(s): 10.0, Count: 26719, OPS: 2666.7, Avg(us): 3704, Min(us): 856, Max(us): 99956, 95th(us): 4000, 99th(us): 84000 [Mon May 6 15:15:06 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50525054.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33064d6c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:4248, start at 2024-05-06 15:15:05.710400209 +0800 CST m=+5.228426088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:05.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:05.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:05.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33064d6c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:4248, start at 2024-05-06 15:15:05.710400209 +0800 CST m=+5.228426088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:05.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:05.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:05.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33064ecc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:4334, start at 2024-05-06 15:15:05.804750873 +0800 CST m=+5.243054236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:05.811 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:05.779 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:05.779 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 13.87 secs (242579330 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Mon May 6 15:15:08 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.56345636.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a542a08e-990e-4292-b89c-f1d53b262df5 {"id":"a542a08e-990e-4292-b89c-f1d53b262df5","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16026f2 a542a08e-990e-4292-b89c-f1d53b262df5 /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a542a08e-990e-4292-b89c-f1d53b262df5 {"id":"a542a08e-990e-4292-b89c-f1d53b262df5","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16026f2 a542a08e-990e-4292-b89c-f1d53b262df5 /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a542a08e-990e-4292-b89c-f1d53b262df5 {"id":"a542a08e-990e-4292-b89c-f1d53b262df5","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16026f2 a542a08e-990e-4292-b89c-f1d53b262df5 /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a {"id":"175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1571ff4 175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a {"id":"175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1571ff4 175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a {"id":"175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1571ff4 175e339b-dbd3-4a4d-a3c6-ac7f8ae3ed0a /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5104.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365781685273812785,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T15:15:09.753681731+08:00","start_ts":449571641000460293,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571641000460293,"checkpoint_ts":449571641000460293,"checkpoint_time":"2024-05-06 15:15:09.627"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "updateproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readproportion"="0" "operationcount"="0" "readallfields"="true" "threadcount"="4" "scanproportion"="0" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "workload"="core" "recordcount"="50" ********************************************** Run finished, takes 16.804699ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3849.1, Avg(us): 1309, Min(us): 815, Max(us): 4350, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later + set +x start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef425f91-045f-46d4-978e-1a40014ff119 {"id":"ef425f91-045f-46d4-978e-1a40014ff119","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16b18ef ef425f91-045f-46d4-978e-1a40014ff119 /tidb/cdc/default/default/upstream/7365781712174540089 {"id":7365781712174540089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef425f91-045f-46d4-978e-1a40014ff119 {"id":"ef425f91-045f-46d4-978e-1a40014ff119","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16b18ef ef425f91-045f-46d4-978e-1a40014ff119 /tidb/cdc/default/default/upstream/7365781712174540089 {"id":7365781712174540089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef425f91-045f-46d4-978e-1a40014ff119 {"id":"ef425f91-045f-46d4-978e-1a40014ff119","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc16b18ef ef425f91-045f-46d4-978e-1a40014ff119 /tidb/cdc/default/default/upstream/7365781712174540089 {"id":7365781712174540089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later The 1 times to try to start tidb cluster... table changefeed_pause_resume.t1 not exists for 1-th check, retry later table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process 5343 exit for 1-th time... wait process 5057 exit for 1-th time... wait process 5057 exit for 2-th time... wait process 5057 exit for 3-th time... wait process 5343 exit for 2-th time... INSERT - Takes(s): 20.0, Count: 35134, OPS: 1755.0, Avg(us): 5660, Min(us): 856, Max(us): 588329, 95th(us): 31000, 99th(us): 89000 wait process 5057 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5343) - No such process wait process 5343 exit for 3-th time... process 5343 already exit check_no_capture http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 5057 exit for 5-th time... parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:15:16 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.55815583.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later wait process 5057 exit for 6-th time... wait process 5057 exit for 7-th time... wait process 5057 exit for 8-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3307014c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:3598, start at 2024-05-06 15:15:17.226777943 +0800 CST m=+6.822485114 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:17.235 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:17.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:17.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3307014c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:3598, start at 2024-05-06 15:15:17.226777943 +0800 CST m=+6.822485114 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:17.235 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:17.203 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:17.203 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330703900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:3676, start at 2024-05-06 15:15:17.395455959 +0800 CST m=+6.909001354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:17.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:17.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:17.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5057 exit for 9-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t3 exists wait process 5057 exit for 10-th time... wait process 5057 exit for 11-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/de352832-6386-450b-91dd-aa1ca710b9a3 {UpstreamID:7365781701185487523 Namespace:default ID:de352832-6386-450b-91dd-aa1ca710b9a3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:15:09.331164853 +0800 CST StartTs:449571640885641224 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f0f560 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571640911855622} {CheckpointTs:449571642078396421 MinTableBarrierTs:449571643651260422 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571642078396421, checkpointTs: 449571642078396421, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5641ff04-4233-4d23-83ff-810b50b61020 {"id":"5641ff04-4233-4d23-83ff-810b50b61020","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc160276d 5641ff04-4233-4d23-83ff-810b50b61020 /tidb/cdc/default/default/changefeed/info/de352832-6386-450b-91dd-aa1ca710b9a3 {"upstream-id":7365781701185487523,"namespace":"default","changefeed-id":"de352832-6386-450b-91dd-aa1ca710b9a3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:15:09.331164853+08:00","start-ts":449571640885641224,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571640911855622} /tidb/cdc/default/default/changefeed/status/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":449571642078396421,"min-table-barrier-ts":449571643651260422,"admin-job-type":0} /tidb/cdc/default/default/task/position/5641ff04-4233-4d23-83ff-810b50b61020/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/de352832-6386-450b-91dd-aa1ca710b9a3 {UpstreamID:7365781701185487523 Namespace:default ID:de352832-6386-450b-91dd-aa1ca710b9a3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:15:09.331164853 +0800 CST StartTs:449571640885641224 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f0f560 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571640911855622} {CheckpointTs:449571642078396421 MinTableBarrierTs:449571643651260422 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571642078396421, checkpointTs: 449571642078396421, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5641ff04-4233-4d23-83ff-810b50b61020 {"id":"5641ff04-4233-4d23-83ff-810b50b61020","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc160276d 5641ff04-4233-4d23-83ff-810b50b61020 /tidb/cdc/default/default/changefeed/info/de352832-6386-450b-91dd-aa1ca710b9a3 {"upstream-id":7365781701185487523,"namespace":"default","changefeed-id":"de352832-6386-450b-91dd-aa1ca710b9a3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:15:09.331164853+08:00","start-ts":449571640885641224,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571640911855622} /tidb/cdc/default/default/changefeed/status/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":449571642078396421,"min-table-barrier-ts":449571643651260422,"admin-job-type":0} /tidb/cdc/default/default/task/position/5641ff04-4233-4d23-83ff-810b50b61020/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/de352832-6386-450b-91dd-aa1ca710b9a3 {UpstreamID:7365781701185487523 Namespace:default ID:de352832-6386-450b-91dd-aa1ca710b9a3 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:15:09.331164853 +0800 CST StartTs:449571640885641224 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f0f560 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571640911855622} {CheckpointTs:449571642078396421 MinTableBarrierTs:449571643651260422 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571642078396421, checkpointTs: 449571642078396421, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5641ff04-4233-4d23-83ff-810b50b61020 {"id":"5641ff04-4233-4d23-83ff-810b50b61020","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc160276d 5641ff04-4233-4d23-83ff-810b50b61020 /tidb/cdc/default/default/changefeed/info/de352832-6386-450b-91dd-aa1ca710b9a3 {"upstream-id":7365781701185487523,"namespace":"default","changefeed-id":"de352832-6386-450b-91dd-aa1ca710b9a3","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:15:09.331164853+08:00","start-ts":449571640885641224,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571640911855622} /tidb/cdc/default/default/changefeed/status/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":449571642078396421,"min-table-barrier-ts":449571643651260422,"admin-job-type":0} /tidb/cdc/default/default/task/position/5641ff04-4233-4d23-83ff-810b50b61020/de352832-6386-450b-91dd-aa1ca710b9a3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781701185487523 {"id":7365781701185487523,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id wait process 5057 exit for 12-th time... check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5035.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5057) - No such process wait process 5057 exit for 13-th time... process 5057 already exit [Mon May 6 15:15:20 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52015203.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "id": "5641ff04-4233-4d23-83ff-810b50b61020", "cluster-id": "default" run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + tso='449571643995455489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571643995455489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x capture_id: 5641ff04-4233-4d23-83ff-810b50b61020 check_processor_table_count http://127.0.0.1:2379 de352832-6386-450b-91dd-aa1ca710b9a3 5641ff04-4233-4d23-83ff-810b50b61020 1 check diff failed 2-th time, retry later run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33074f980011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:3779, start at 2024-05-06 15:15:22.236494419 +0800 CST m=+5.271508609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:22.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:22.214 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:22.214 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:15:23 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.50735075.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_processor_table_count http://127.0.0.1:2379 de352832-6386-450b-91dd-aa1ca710b9a3 5641ff04-4233-4d23-83ff-810b50b61020 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 14.60 secs (230478498 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33074f980011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:3779, start at 2024-05-06 15:15:22.236494419 +0800 CST m=+5.271508609 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:22.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:22.214 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:22.214 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33074f900013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:3866, start at 2024-05-06 15:15:22.244760287 +0800 CST m=+5.202333966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:22.253 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:22.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:22.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_processor_table_count http://127.0.0.1:2379 de352832-6386-450b-91dd-aa1ca710b9a3 5641ff04-4233-4d23-83ff-810b50b61020 0 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641432997924,"min-table-barrier-ts":449571641432997924,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641432997924,"min-table-barrier-ts":449571641432997924,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641432997924,"min-table-barrier-ts":449571641432997924,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:15:25 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52625264.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > run task successfully INSERT - Takes(s): 30.0, Count: 44932, OPS: 1496.8, Avg(us): 6643, Min(us): 856, Max(us): 588329, 95th(us): 76000, 99th(us): 89000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f852cfa-9f93-4374-93c1-9a0d2b02b459 {"id":"8f852cfa-9f93-4374-93c1-9a0d2b02b459","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc18de710 8f852cfa-9f93-4374-93c1-9a0d2b02b459 /tidb/cdc/default/default/upstream/7365781758839381857 {"id":7365781758839381857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f852cfa-9f93-4374-93c1-9a0d2b02b459 {"id":"8f852cfa-9f93-4374-93c1-9a0d2b02b459","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc18de710 8f852cfa-9f93-4374-93c1-9a0d2b02b459 /tidb/cdc/default/default/upstream/7365781758839381857 {"id":7365781758839381857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8f852cfa-9f93-4374-93c1-9a0d2b02b459 {"id":"8f852cfa-9f93-4374-93c1-9a0d2b02b459","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc18de710 8f852cfa-9f93-4374-93c1-9a0d2b02b459 /tidb/cdc/default/default/upstream/7365781758839381857 {"id":7365781758839381857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5127.out cli changefeed create --start-ts=449571643995455489 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 51faeb86-f2c4-4ec3-9a24-224da6da39b7 Info: {"upstream_id":7365781758839381857,"namespace":"default","id":"51faeb86-f2c4-4ec3-9a24-224da6da39b7","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:15:27.119369698+08:00","start_ts":449571643995455489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571643995455489,"checkpoint_ts":449571643995455489,"checkpoint_time":"2024-05-06 15:15:21.052"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="changefeed_reconstruct" "mysql.port"="4000" "dotransactions"="false" "requestdistribution"="uniform" "mysql.user"="root" "recordcount"="50" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "workload"="core" "readproportion"="0" "threadcount"="4" "readallfields"="true" ********************************************** Run finished, takes 18.483172ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3525.5, Avg(us): 1473, Min(us): 975, Max(us): 5109, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641458950179,"min-table-barrier-ts":449571641458950179,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641458950179,"min-table-barrier-ts":449571641458950179,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365781685273812785 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:15:09.753681731 +0800 CST StartTs:449571641000460293 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038a95f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571641026412549} {CheckpointTs:449571641432997924 MinTableBarrierTs:449571641432997924 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/950b650d-ed88-46e7-8ffc-c21a100db4c3 {"id":"950b650d-ed88-46e7-8ffc-c21a100db4c3","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc15721a1 950b650d-ed88-46e7-8ffc-c21a100db4c3 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365781685273812785,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:15:09.753681731+08:00","start-ts":449571641000460293,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571641026412549} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449571641458950179,"min-table-barrier-ts":449571641458950179,"admin-job-type":0} /tidb/cdc/default/default/task/position/950b650d-ed88-46e7-8ffc-c21a100db4c3/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781685273812785 {"id":7365781685273812785,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5187.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff failed 5-th time, retry later + set +x + tso='449571645715906561 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571645715906561 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:15:29 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52215223.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:15:29 CST 2024] <<<<<< run test case sink_retry success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3307bde8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:4082, start at 2024-05-06 15:15:29.28659105 +0800 CST m=+5.116885635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:29.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:29.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:29.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3307bde8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:4082, start at 2024-05-06 15:15:29.28659105 +0800 CST m=+5.116885635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:29.292 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:29.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:29.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3307bdec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:4171, start at 2024-05-06 15:15:29.323225634 +0800 CST m=+5.071348244 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:29.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:29.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:29.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... check diff successfully pass test: get tso wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:15:30 CST 2024] <<<<<< run test case http_api success! >>>>>> Run finished, takes 35.557045667s INSERT - Takes(s): 35.6, Count: 50000, OPS: 1406.4, Avg(us): 7035, Min(us): 856, Max(us): 588329, 95th(us): 78000, 99th(us): 89000 table consistent_replicate_nfs.usertable exists cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:15:31 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> table ddl_manager.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1efc06d3-83b8-4aed-b068-c53ec9554e43 {"id":"1efc06d3-83b8-4aed-b068-c53ec9554e43","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1a9b1f6 1efc06d3-83b8-4aed-b068-c53ec9554e43 /tidb/cdc/default/default/upstream/7365781783609501687 {"id":7365781783609501687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1efc06d3-83b8-4aed-b068-c53ec9554e43 {"id":"1efc06d3-83b8-4aed-b068-c53ec9554e43","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1a9b1f6 1efc06d3-83b8-4aed-b068-c53ec9554e43 /tidb/cdc/default/default/upstream/7365781783609501687 {"id":7365781783609501687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1efc06d3-83b8-4aed-b068-c53ec9554e43 {"id":"1efc06d3-83b8-4aed-b068-c53ec9554e43","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1a9b1f6 1efc06d3-83b8-4aed-b068-c53ec9554e43 /tidb/cdc/default/default/upstream/7365781783609501687 {"id":7365781783609501687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5270.out cli changefeed create --start-ts=449571645715906561 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 8405648f-f67c-44ae-8d4e-6c0ff1eb2308 [Mon May 6 15:15:32 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.54065408.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Info: {"upstream_id":7365781783609501687,"namespace":"default","id":"8405648f-f67c-44ae-8d4e-6c0ff1eb2308","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:15:32.505699995+08:00","start_ts":449571645715906561,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571645715906561,"checkpoint_ts":449571645715906561,"checkpoint_time":"2024-05-06 15:15:27.615"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit table ddl_manager.finish_mark not exists for 4-th check, retry later + set +x table drop_tables.c not exists for 1-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:15:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d77e1fd9-3c9e-4de3-935f-26aefbdf2067 {"id":"d77e1fd9-3c9e-4de3-935f-26aefbdf2067","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1cb84f1 d77e1fd9-3c9e-4de3-935f-26aefbdf2067 /tidb/cdc/default/default/upstream/7365781816828752540 {"id":7365781816828752540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d77e1fd9-3c9e-4de3-935f-26aefbdf2067 {"id":"d77e1fd9-3c9e-4de3-935f-26aefbdf2067","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1cb84f1 d77e1fd9-3c9e-4de3-935f-26aefbdf2067 /tidb/cdc/default/default/upstream/7365781816828752540 {"id":7365781816828752540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d77e1fd9-3c9e-4de3-935f-26aefbdf2067 {"id":"d77e1fd9-3c9e-4de3-935f-26aefbdf2067","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc1cb84f1 d77e1fd9-3c9e-4de3-935f-26aefbdf2067 /tidb/cdc/default/default/upstream/7365781816828752540 {"id":7365781816828752540,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5458.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 768e8329-84d1-42da-b93d-d9e75fd98d00 Info: {"upstream_id":7365781816828752540,"namespace":"default","id":"768e8329-84d1-42da-b93d-d9e75fd98d00","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:15:35.998658388+08:00","start_ts":449571647881216003,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571647881216003,"checkpoint_ts":449571647881216003,"checkpoint_time":"2024-05-06 15:15:35.875"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table drop_tables.c not exists for 2-th check, retry later check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 9.32 secs (361112052 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 6-th check, retry later + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading golang.org/x/text v0.14.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table drop_tables.c not exists for 3-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/otiai10/copy v1.2.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 table drop_tables.c not exists for 4-th check, retry later check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 8-th check, retry later table drop_tables.c not exists for 5-th check, retry later check diff successfully table ddl_manager.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table drop_tables.c exists check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449571646151065607" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 15:15:43.711 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:15:43.712 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:15:43.712 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571644014329949] [resolvedTs=449571646151065607] [2024/05/06 15:15:43.712 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=330.901ยตs] [2024/05/06 15:15:43.713 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:15:43.713 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:15:43.718 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:15:43.721 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:15:43.721 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:15:43.723 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:15:43.723 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:15:43.726 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:15:43.728 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 15:15:43.751 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=39.566441ms] [2024/05/06 15:15:44.552 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}] [checkpointTs=449571646151065607] [2024/05/06 15:15:44.552 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}] [checkpointTs=449571646151065607] [2024/05/06 15:15:44.652 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=7898] [appliedDDLCount=0] [currentCheckpoint=449571646151065607] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:15:44.652 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] Apply redo log successfully check diff successfully [Mon May 6 15:15:44 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 48133 table ddl_manager.finish_mark not exists for 10-th check, retry later go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:15:46 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 11-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table ddl_manager.finish_mark not exists for 13-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330937a00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:3724, start at 2024-05-06 15:15:53.450968042 +0800 CST m=+5.216671647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.458 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330937a00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:3724, start at 2024-05-06 15:15:53.450968042 +0800 CST m=+5.216671647 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.458 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330939140001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:3797, start at 2024-05-06 15:15:53.542073503 +0800 CST m=+5.256028204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33093d040012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:4110, start at 2024-05-06 15:15:53.825496559 +0800 CST m=+5.268417350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully table ddl_manager.finish_mark not exists for 15-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5171.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33093d040012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:4110, start at 2024-05-06 15:15:53.825496559 +0800 CST m=+5.268417350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33093e6c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:4193, start at 2024-05-06 15:15:53.905987526 +0800 CST m=+5.300908921 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:53.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:53.883 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:53.883 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later ____________________________________ *************************** 1. row *************************** primary_ts: 449571589326372867 secondary_ts: 449571591970881549 *************************** 2. row *************************** primary_ts: 449571597190692864 secondary_ts: 449571597489012737 *************************** 3. row *************************** primary_ts: 449571605055012864 secondary_ts: 449571605379547137 *************************** 4. row *************************** primary_ts: 449571612919332864 secondary_ts: 449571613244129281 *************************** 5. row *************************** primary_ts: 449571620783652864 secondary_ts: 449571621108187137 *************************** 6. row *************************** primary_ts: 449571628647972864 secondary_ts: 449571628736839681 *************************** 7. row *************************** primary_ts: 449571636512292864 secondary_ts: 449571636601159681 *************************** 8. row *************************** primary_ts: 449571644376612864 secondary_ts: 449571644465479681 skip invalid syncpoint primary_ts: 449571589326372867, first_ddl_ts: 449571592721399835 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:15:52 CST 2024] <<<<<< run test case syncpoint success! >>>>>> table ddl_manager.finish_mark not exists for 16-th check, retry later + set +x + tso='449571653379162113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571653379162113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330970b00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:6835, start at 2024-05-06 15:15:57.123390457 +0800 CST m=+5.078268306 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:57.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:57.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:57.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330970b00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:6835, start at 2024-05-06 15:15:57.123390457 +0800 CST m=+5.078268306 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:57.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:57.100 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:57.100 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330973040013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6shct, pid:6917, start at 2024-05-06 15:15:57.278579184 +0800 CST m=+5.184482808 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:17:57.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:15:57.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:05:57.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 check diff failed 2-th time, retry later ***************** properties ***************** "readallfields"="true" "mysql.db"="changefeed_error" "readproportion"="0" "scanproportion"="0" "insertproportion"="0" "mysql.port"="4000" "workload"="core" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "recordcount"="20" "requestdistribution"="uniform" "operationcount"="0" "threadcount"="4" ********************************************** table ddl_manager.finish_mark not exists for 17-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8231.out cli tso query --pd=http://127.0.0.1:2379 Run finished, takes 1.725597545s INSERT - Takes(s): 1.7, Count: 20, OPS: 11.6, Avg(us): 173490, Min(us): 872, Max(us): 1717675, 95th(us): 0, 99th(us): 0 [Mon May 6 15:16:01 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.52405242.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:01 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 18-th check, retry later Starting TLS TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 21.48 secs (156618858 bytes/sec) [Pipeline] { [Pipeline] cache + set +x + tso='449571654388940801 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571654388940801 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.port"="4000" "readallfields"="true" "scanproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10" "operationcount"="0" "workload"="core" "threadcount"="2" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "mysql.user"="root" "mysql.db"="multi_capture_1" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.895981ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2119.7, Avg(us): 1895, Min(us): 870, Max(us): 5360, 95th(us): 6000, 99th(us): 6000 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... check diff failed 4-th time, retry later Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 19-th check, retry later Starting TLS TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 {"id":"3d5db36d-45a7-4f68-bdae-eb68b83be825","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f970e 3d5db36d-45a7-4f68-bdae-eb68b83be825 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 {"id":"3d5db36d-45a7-4f68-bdae-eb68b83be825","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f970e 3d5db36d-45a7-4f68-bdae-eb68b83be825 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 {"id":"3d5db36d-45a7-4f68-bdae-eb68b83be825","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f970e 3d5db36d-45a7-4f68-bdae-eb68b83be825 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5294.out cli changefeed create --start-ts=449571653379162113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error ***************** properties ***************** "mysql.db"="multi_capture_2" "insertproportion"="0" "mysql.port"="4000" "operationcount"="0" "mysql.host"="127.0.0.1" "workload"="core" "requestdistribution"="uniform" "dotransactions"="false" "updateproportion"="0" "readproportion"="0" "scanproportion"="0" "recordcount"="10" "readallfields"="true" "threadcount"="2" "mysql.user"="root" ********************************************** Run finished, takes 10.95497ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1868.3, Avg(us): 2105, Min(us): 1111, Max(us): 5503, 95th(us): 6000, 99th(us): 6000 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365781914866356277,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:16:04.542751861+08:00","start_ts":449571653379162113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571653379162113,"checkpoint_ts":449571653379162113,"checkpoint_time":"2024-05-06 15:15:56.848"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later ***************** properties ***************** "threadcount"="2" "mysql.user"="root" "requestdistribution"="uniform" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "mysql.db"="multi_capture_3" "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "scanproportion"="0" "workload"="core" "readallfields"="true" "recordcount"="10" ********************************************** Run finished, takes 9.991269ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2014.7, Avg(us): 1918, Min(us): 1021, Max(us): 4973, 95th(us): 5000, 99th(us): 5000 table ddl_manager.finish_mark not exists for 20-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449571653379162113, "checkpoint_time": "2024-05-06 15:15:56.848", "error": { "time": "2024-05-06T15:16:04.65064744+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at 449571653379162113" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449571653379162113, "checkpoint_time": "2024-05-06 15:15:56.848", "error": { "time": "2024-05-06T15:16:04.65064744+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at 449571653379162113" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449571653379162113, "checkpoint_time": "2024-05-06 15:15:56.848", "error": { "time": "2024-05-06T15:16:04.65064744+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at 449571653379162113" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449571653379162113, '"checkpoint_time":' '"2024-05-06' '15:15:56.848",' '"error":' '{' '"time":' '"2024-05-06T15:16:04.65064744+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at '449571653379162113"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449571653379162113, '"checkpoint_time":' '"2024-05-06' '15:15:56.848",' '"error":' '{' '"time":' '"2024-05-06T15:16:04.65064744+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at '449571653379162113"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at 449571653379162113' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449571653379161813 is earlier than or equal to GC safepoint at 449571653379162113 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5364.out cli changefeed resume -c changefeed-error ***************** properties ***************** "recordcount"="10" "workload"="core" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_capture_4" "threadcount"="2" "readallfields"="true" "dotransactions"="false" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 10.35845ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1986.4, Avg(us): 1985, Min(us): 1091, Max(us): 5229, 95th(us): 6000, 99th(us): 6000 [Mon May 6 15:16:06 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83638365.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 21-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + set +x table changefeed_error.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:16:09 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.84098411.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 22-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a29d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:5718, start at 2024-05-06 15:16:08.997131742 +0800 CST m=+5.193982670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:09.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:08.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:08.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Mon May 6 15:16:09 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.57735775.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_error.usertable exists check diff successfully ***************** properties ***************** "mysql.port"="4000" "updateproportion"="0" "mysql.user"="root" "threadcount"="4" "operationcount"="0" "readproportion"="0" "requestdistribution"="uniform" "recordcount"="20" "readallfields"="true" "mysql.host"="127.0.0.1" "workload"="core" "scanproportion"="0" "dotransactions"="false" "insertproportion"="0" "mysql.db"="changefeed_error" ********************************************** Run finished, takes 5.098918ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5422.6, Avg(us): 869, Min(us): 490, Max(us): 1949, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"3d5db36d-45a7-4f68-bdae-eb68b83be825","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + [[ /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + [[ /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:12 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:16:12 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.84588460.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a57bc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:9604, start at 2024-05-06 15:16:11.89297732 +0800 CST m=+5.315499249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:11.901 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:11.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:11.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:13 GMT < Content-Length: 1067 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a57bc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:9604, start at 2024-05-06 15:16:11.89297732 +0800 CST m=+5.315499249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:11.901 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:11.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:11.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a58700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:9678, start at 2024-05-06 15:16:11.951279322 +0800 CST m=+5.312191099 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:11.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:11.932 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:11.932 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later [Mon May 6 15:16:15 CST 2024] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.58335835.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 25-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:15 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d55af31-7cbb-4dfe-a06a-7815bca3e805 {"id":"2d55af31-7cbb-4dfe-a06a-7815bca3e805","address":"127.0.0.1:8303","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367588 2d55af31-7cbb-4dfe-a06a-7815bca3e805 /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d55af31-7cbb-4dfe-a06a-7815bca3e805 {"id":"2d55af31-7cbb-4dfe-a06a-7815bca3e805","address":"127.0.0.1:8303","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367588 2d55af31-7cbb-4dfe-a06a-7815bca3e805 /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d55af31-7cbb-4dfe-a06a-7815bca3e805 {"id":"2d55af31-7cbb-4dfe-a06a-7815bca3e805","address":"127.0.0.1:8303","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/4da65bd3-ed8f-4a2b-85a3-5487c074e7df {"id":"4da65bd3-ed8f-4a2b-85a3-5487c074e7df","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/b2aa57f0-fd03-4e44-961e-286ab802d4ac {"id":"b2aa57f0-fd03-4e44-961e-286ab802d4ac","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc236754e 4da65bd3-ed8f-4a2b-85a3-5487c074e7df /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367577 b2aa57f0-fd03-4e44-961e-286ab802d4ac /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2367588 2d55af31-7cbb-4dfe-a06a-7815bca3e805 /tidb/cdc/default/default/upstream/7365781931781242754 {"id":7365781931781242754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8516.out cli changefeed create --start-ts=449571654388940801 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: c5df36ce-6844-4a1a-a212-cda863687929 Info: {"upstream_id":7365781931781242754,"namespace":"default","id":"c5df36ce-6844-4a1a-a212-cda863687929","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:16:15.921617965+08:00","start_ts":449571654388940801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571654388940801,"checkpoint_ts":449571654388940801,"checkpoint_time":"2024-05-06 15:16:00.700"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a84040013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:6239, start at 2024-05-06 15:16:14.757504862 +0800 CST m=+5.191201956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:14.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:14.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:14.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a84040013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:6239, start at 2024-05-06 15:16:14.757504862 +0800 CST m=+5.191201956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:14.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:14.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:14.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330a857c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:6310, start at 2024-05-06 15:16:14.826754255 +0800 CST m=+5.196242994 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:14.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:14.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:14.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + [[ /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/3d5db36d-45a7-4f68-bdae-eb68b83be825 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later [Mon May 6 15:16:16 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1100611008.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later + set +x table multi_capture_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 26-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > [Mon May 6 15:16:18 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.76737675.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:18 GMT < Content-Length: 1321 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c980d04-9c86-4ff3-8362-8d87f138d52c {"id":"1c980d04-9c86-4ff3-8362-8d87f138d52c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a3c 1c980d04-9c86-4ff3-8362-8d87f138d52c /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c980d04-9c86-4ff3-8362-8d87f138d52c {"id":"1c980d04-9c86-4ff3-8362-8d87f138d52c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a3c 1c980d04-9c86-4ff3-8362-8d87f138d52c /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c980d04-9c86-4ff3-8362-8d87f138d52c {"id":"1c980d04-9c86-4ff3-8362-8d87f138d52c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e {"id":"e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a0f e0d62ce9-48a9-4d2a-b9db-4026ca5bd20e /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4cc2638a3c 1c980d04-9c86-4ff3-8362-8d87f138d52c /tidb/cdc/default/default/upstream/7365781983651475560 {"id":7365781983651475560,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330ac1f40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:10184, start at 2024-05-06 15:16:18.704354541 +0800 CST m=+5.280791006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:18.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:18.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:18.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b1da665-5f81-404e-9661-e4ab0272840e {"id":"2b1da665-5f81-404e-9661-e4ab0272840e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d46f0 2b1da665-5f81-404e-9661-e4ab0272840e /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b1da665-5f81-404e-9661-e4ab0272840e {"id":"2b1da665-5f81-404e-9661-e4ab0272840e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d46f0 2b1da665-5f81-404e-9661-e4ab0272840e /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2b1da665-5f81-404e-9661-e4ab0272840e {"id":"2b1da665-5f81-404e-9661-e4ab0272840e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d46f0 2b1da665-5f81-404e-9661-e4ab0272840e /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "readallfields"="true" "recordcount"="5000" "readproportion"="0" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.db"="consistent_replicate_storage_file" "workload"="core" "fieldcount"="100" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" "updateproportion"="0" "operationcount"="0" "threadcount"="10" "mysql.host"="127.0.0.1" "scanproportion"="0" ********************************************** pass test: check health pass test: get status check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b13055-0d12-4790-a577-49ef38fe46ce {"id":"97b13055-0d12-4790-a577-49ef38fe46ce","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc27509f2 97b13055-0d12-4790-a577-49ef38fe46ce /tidb/cdc/default/default/upstream/7365782011695902461 {"id":7365782011695902461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b13055-0d12-4790-a577-49ef38fe46ce {"id":"97b13055-0d12-4790-a577-49ef38fe46ce","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc27509f2 97b13055-0d12-4790-a577-49ef38fe46ce /tidb/cdc/default/default/upstream/7365782011695902461 {"id":7365782011695902461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97b13055-0d12-4790-a577-49ef38fe46ce {"id":"97b13055-0d12-4790-a577-49ef38fe46ce","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc27509f2 97b13055-0d12-4790-a577-49ef38fe46ce /tidb/cdc/default/default/upstream/7365782011695902461 {"id":7365782011695902461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449571659670618121, "checkpoint_time": "2024-05-06 15:16:20.848", "error": null }' + echo '{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449571659670618121, "checkpoint_time": "2024-05-06 15:16:20.848", "error": null }' { "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 449571659670618121, "checkpoint_time": "2024-05-06 15:16:20.848", "error": null } ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449571659670618121, '"checkpoint_time":' '"2024-05-06' '15:16:20.848",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 449571659670618121, '"checkpoint_time":' '"2024-05-06' '15:16:20.848",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330ac1f40011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:10184, start at 2024-05-06 15:16:18.704354541 +0800 CST m=+5.280791006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:18.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:18.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:18.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330ac3640004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qmjhq, pid:10258, start at 2024-05-06 15:16:18.780857391 +0800 CST m=+5.299232658 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:18.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:18.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:18.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + info='{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449571659723309065, "checkpoint_time": "2024-05-06 15:16:21.049", "error": null }' + echo '{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449571659723309065, "checkpoint_time": "2024-05-06 15:16:21.049", "error": null }' { "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 449571659723309065, "checkpoint_time": "2024-05-06 15:16:21.049", "error": null } ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449571659723309065, '"checkpoint_time":' '"2024-05-06' '15:16:21.049",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 449571659723309065, '"checkpoint_time":' '"2024-05-06' '15:16:21.049",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449571659762368523, "checkpoint_time": "2024-05-06 15:16:21.198", "error": null }' + echo '{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449571659762368523, "checkpoint_time": "2024-05-06 15:16:21.198", "error": null }' { "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 449571659762368523, "checkpoint_time": "2024-05-06 15:16:21.198", "error": null } ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449571659762368523, '"checkpoint_time":' '"2024-05-06' '15:16:21.198",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 449571659762368523, '"checkpoint_time":' '"2024-05-06' '15:16:21.198",' '"error":' null '}' ++ jq -r .error.message Run finished, takes 1.872313797s INSERT - Takes(s): 1.9, Count: 5000, OPS: 2683.3, Avg(us): 3408, Min(us): 1534, Max(us): 56794, 95th(us): 4000, 99th(us): 36000 + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449571659815059465, "checkpoint_time": "2024-05-06 15:16:21.399", "error": null }' + echo '{ "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449571659815059465, "checkpoint_time": "2024-05-06 15:16:21.399", "error": null }' { "upstream_id": 7365781983651475560, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 449571659815059465, "checkpoint_time": "2024-05-06 15:16:21.399", "error": null } ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449571659815059465, '"checkpoint_time":' '"2024-05-06' '15:16:21.399",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781983651475560, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 449571659815059465, '"checkpoint_time":' '"2024-05-06' '15:16:21.399",' '"error":' null '}' ++ jq -r .error.message table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later + message=null + [[ ! null =~ null ]] run task successfully table processor_stop_delay.t not exists for 1-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 15:16:22 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.55695571.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11509.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 29-th check, retry later table processor_stop_delay.t exists check diff failed 1-th time, retry later table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "updateproportion"="0" "recordcount"="20" "scanproportion"="0" "mysql.db"="multi_capture_1" "readproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readallfields"="true" "operationcount"="0" "threadcount"="2" "workload"="core" "mysql.port"="4000" "insertproportion"="0" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 11.907887ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1945.9, Avg(us): 1122, Min(us): 490, Max(us): 1982, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "operationcount"="0" "mysql.user"="root" "dotransactions"="false" "threadcount"="2" "readproportion"="0" "readallfields"="true" "insertproportion"="0" "updateproportion"="0" "mysql.db"="multi_capture_2" "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="20" "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" ********************************************** Run finished, takes 11.600705ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2006.5, Avg(us): 1098, Min(us): 534, Max(us): 1792, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "workload"="core" "mysql.db"="multi_capture_3" "readallfields"="true" "threadcount"="2" "recordcount"="20" "updateproportion"="0" "readproportion"="0" "scanproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.port"="4000" "operationcount"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** Run finished, takes 11.592108ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1946.8, Avg(us): 1100, Min(us): 491, Max(us): 1804, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "updateproportion"="0" "threadcount"="2" "recordcount"="20" "workload"="core" "mysql.db"="multi_capture_4" "scanproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "insertproportion"="0" "readallfields"="true" "readproportion"="0" ********************************************** Run finished, takes 10.956958ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2144.3, Avg(us): 1038, Min(us): 472, Max(us): 2053, 95th(us): 2000, 99th(us): 3000 + (( i++ )) + set +x + tso='449571660506071041 PASS check diff failed 3-th time, retry later coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + (( i <= 50 )) + echo 449571660506071041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + awk -F ' ' '{print $1}' + set +x * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) [Mon May 6 15:16:25 CST 2024] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' > Host: 127.0.0.1:8300 > Accept: */* > + '[' -z '' ']' < HTTP/1.1 200 OK + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] < Date: Mon, 06 May 2024 07:16:25 GMT + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4670b4dd-017c-4966-bc2f-4a1078c42e38 {"id":"4670b4dd-017c-4966-bc2f-4a1078c42e38","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f978b 4670b4dd-017c-4966-bc2f-4a1078c42e38 /tidb/cdc/default/default/changefeed/info/changefeed-error + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1156011562.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 {"upstream-id":7365781914866356277,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:16:04.542751861+08:00","start-ts":449571653379162113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T15:16:22.922471665+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571660208013320} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449571656760819717,"min-table-barrier-ts":449571656760819719,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4670b4dd-017c-4966-bc2f-4a1078c42e38 {"id":"4670b4dd-017c-4966-bc2f-4a1078c42e38","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f978b 4670b4dd-017c-4966-bc2f-4a1078c42e38 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365781914866356277,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:16:04.542751861+08:00","start-ts":449571653379162113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T15:16:22.922471665+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571660208013320} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449571656760819717,"min-table-barrier-ts":449571656760819719,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365781914866356277 + grep -q 'failed to get info:' {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4670b4dd-017c-4966-bc2f-4a1078c42e38 {"id":"4670b4dd-017c-4966-bc2f-4a1078c42e38","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f978b 4670b4dd-017c-4966-bc2f-4a1078c42e38 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365781914866356277,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:16:04.542751861+08:00","start-ts":449571653379162113,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T15:16:22.922471665+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571660208013320} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449571656760819717,"min-table-barrier-ts":449571656760819719,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449571656760819717, "checkpoint_time": "2024-05-06 15:16:09.748", "error": { "time": "2024-05-06T15:16:22.922471665+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449571656760819717, "checkpoint_time": "2024-05-06 15:16:09.748", "error": { "time": "2024-05-06T15:16:22.922471665+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449571656760819717, "checkpoint_time": "2024-05-06 15:16:09.748", "error": { "time": "2024-05-06T15:16:22.922471665+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571656760819717, '"checkpoint_time":' '"2024-05-06' '15:16:09.748",' '"error":' '{' '"time":' '"2024-05-06T15:16:22.922471665+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state check diff failed 1-th time, retry later + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571656760819717, '"checkpoint_time":' '"2024-05-06' '15:16:09.748",' '"error":' '{' '"time":' '"2024-05-06T15:16:22.922471665+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5658.out cli changefeed remove -c changefeed-error table ddl_manager.finish_mark not exists for 30-th check, retry later table test.simple1 exists pass test: list changefeed Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449571656760819717 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully pass test: get changefeed check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later pass test: pause changefeed wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 {"id":"f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2842107 f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 /tidb/cdc/default/default/upstream/7365782026846604844 {"id":7365782026846604844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 {"id":"f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2842107 f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 /tidb/cdc/default/default/upstream/7365782026846604844 {"id":7365782026846604844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 {"id":"f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2842107 f9fb5dbd-0fb7-4f8f-958d-7a0d6f0ee359 /tidb/cdc/default/default/upstream/7365782026846604844 {"id":7365782026846604844,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11610.out cli changefeed create --start-ts=449571660506071041 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 12af2876-5a71-4a5d-9fcb-c9053b400208 Info: {"upstream_id":7365782026846604844,"namespace":"default","id":"12af2876-5a71-4a5d-9fcb-c9053b400208","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:16:29.153718038+08:00","start_ts":449571660506071041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571660506071041,"checkpoint_ts":449571660506071041,"checkpoint_time":"2024-05-06 15:16:24.035"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:16:29 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 32-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 15:16:29 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57415743.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later + set +x TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later pass test: update changefeed check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later check diff successfully pass test: resume changefeed table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later table test.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:33 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9ab6a61-dea2-4436-bfe9-d8acdf462348 {"id":"a9ab6a61-dea2-4436-bfe9-d8acdf462348","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f97c8 a9ab6a61-dea2-4436-bfe9-d8acdf462348 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9ab6a61-dea2-4436-bfe9-d8acdf462348 {"id":"a9ab6a61-dea2-4436-bfe9-d8acdf462348","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f97c8 a9ab6a61-dea2-4436-bfe9-d8acdf462348 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a9ab6a61-dea2-4436-bfe9-d8acdf462348 {"id":"a9ab6a61-dea2-4436-bfe9-d8acdf462348","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f97c8 a9ab6a61-dea2-4436-bfe9-d8acdf462348 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5786.out cli changefeed create --start-ts=449571653379162113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc) 3364945408 bytes in 27.31 secs (123230325 bytes/sec) [Pipeline] { TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later [Pipeline] sh Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365781914866356277,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:16:33.390437838+08:00","start_ts":449571653379162113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571653379162113,"checkpoint_ts":449571653379162113,"checkpoint_time":"2024-05-06 15:15:56.848"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later pass test: rebalance table table ddl_manager.finish_mark not exists for 34-th check, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=7e5508ee-9523-401e-aa50-75a99951f897 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-h7j4j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-h7j4j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=0ffddeed-c91d-4d54-8477-2fecf9ea26a4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-bvb0p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-bvb0p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh pass test: list processors + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=55c96964-4eba-4eeb-aae9-f0a8e68fec39 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k2146 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k2146 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 6-th check, retry later [Pipeline] sh table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later + set +x + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: big_txn changefeed_finish force_replicate_table PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=a08764d4-183e-41ab-b07e-3b17a0cc8625 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k1v9z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-k1v9z GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 4-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=2f029c4d-14da-4a5b-ad69-74f86200cb79 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-7dps8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-7dps8 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh check data successfully check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=4c61fbb0-df15-4b4c-a4ad-2d8dd62adff4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-cn3gg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-cn3gg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... [Pipeline] sh 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 2153 0 --:--:-- --:--:-- --:--:-- 2172 + info='{"state":"warning","resolved_ts":449571653759270935,"checkpoint_ts":449571653759270935,"last_warning":{"time":"2024-05-06T15:16:35.01756311+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449571653759270935,"checkpoint_ts":449571653759270935,"last_warning":{"time":"2024-05-06T15:16:35.01756311+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449571653759270935,"checkpoint_ts":449571653759270935,"last_warning":{"time":"2024-05-06T15:16:35.01756311+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449571653759270935,"checkpoint_ts":449571653759270935,"last_warning":{"time":"2024-05-06T15:16:35.01756311+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449571653759270935,"checkpoint_ts":449571653759270935,"last_warning":{"time":"2024-05-06T15:16:35.01756311+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5836.out cli changefeed remove -c changefeed-error-1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=f4a1dd0f-977a-4fbf-9dba-860f8557afa4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-c7g74 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-c7g74 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=e8b40044-3498-4bee-b0ea-7793759f3f61 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-gx8nm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-gx8nm ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 35-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=f7863624-437b-4c8e-b5fb-e5d1202ed584 JENKINS_NODE_COOKIE=6ca69a04-5baa-4b41-9b76-3cf1cc4418ad BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787377995250405378","prowjobid":"f7863624-437b-4c8e-b5fb-e5d1202ed584","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)","link":"https://github.com/pingcap/tiflow/pull/11000","commit_link":"https://github.com/pingcap/tiflow/pull/11000/commits/b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#361 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787377995250405378 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/361/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-hvd30 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_361-hvd30 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=361 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= table test.finish_mark not exists for 7-th check, retry later Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449571653759270935 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:16:36 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table test.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later pass test: get processors wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 15:16:39 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58855887.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully pass test: move table wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later table test.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:16:40 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> pass test: set log level start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 38-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... pass test: remove changefeed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:42 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/062f5360-c8f6-48b4-8ab5-4ba4f409ea36 {"id":"062f5360-c8f6-48b4-8ab5-4ba4f409ea36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f9816 062f5360-c8f6-48b4-8ab5-4ba4f409ea36 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/062f5360-c8f6-48b4-8ab5-4ba4f409ea36 {"id":"062f5360-c8f6-48b4-8ab5-4ba4f409ea36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f9816 062f5360-c8f6-48b4-8ab5-4ba4f409ea36 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/062f5360-c8f6-48b4-8ab5-4ba4f409ea36 {"id":"062f5360-c8f6-48b4-8ab5-4ba4f409ea36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f9816 062f5360-c8f6-48b4-8ab5-4ba4f409ea36 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5930.out cli changefeed create --start-ts=449571653379162113 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365781914866356277,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:16:42.770852239+08:00","start_ts":449571653379162113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571653379162113,"checkpoint_ts":449571653379162113,"checkpoint_time":"2024-05-06 15:15:56.848"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later pass test: resign owner table ddl_manager.finish_mark not exists for 39-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m check diff successfully pass test: get tso + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:16:42.779707433+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:16:42.779707433+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:16:42.779707433+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T15:16:42.779707433+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T15:16:42.779707433+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449571653379162113 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5994.out cli changefeed remove -c changefeed-error-2 start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 40-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: verify table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:16:32.394 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/06 15:16:32.394 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/06 15:16:32.695 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 15:16:32.798 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 15:16:32.917 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1f0b5f2c_5ed3_4dd0_b40d_0cc75898cb4f"] [2024/05/06 15:16:32.924 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0ba4a08b_ea15_4fd8_8c85_c379893057fb"] [2024/05/06 15:16:32.925 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs93ffc232_4b2c_4608_a1f6_e3eb427c862d"] [2024/05/06 15:16:32.926 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa0c8e841_71c6_4d22_aa77_9516e5f1d396"] [2024/05/06 15:16:32.928 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7d1eb2ad_8334_44ca_8cfe_7ae0c8a780fd"] [2024/05/06 15:16:32.929 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0056841f_0a7a_4965_a4a9_817e4c42a25b"] [2024/05/06 15:16:32.995 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs20d0605b_e223_4b36_8303_36c500cc9704"] [2024/05/06 15:16:33.013 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsee930920_cdb5_4619_b44d_1cecf088bd25"] [2024/05/06 15:16:33.021 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 15:16:33.118 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 15:16:33.401 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 15:16:34.096 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.096 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.102 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.109 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.110 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.117 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 15:16:34.118 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.120 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.122 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 15:16:34.200 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.215 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.215 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.216 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:34.218 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.219 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.228 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.230 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.312 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:34.317 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 15:16:34.400 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 15:16:35.017 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 15:16:35.024 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 15:16:35.222 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.294 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.309 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.311 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.316 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.321 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.397 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.409 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.410 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.413 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:35.415 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.418 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.420 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.497 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.502 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.511 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:35.521 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 15:16:35.597 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 15:16:35.797 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 15:16:36.409 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.412 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.422 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.497 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.500 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.500 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.507 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.510 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.514 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.516 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.518 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.520 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:16:36.522 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.526 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.597 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.604 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 15:16:36.607 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:16:36.613 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 15:16:36.617 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 15:16:37.314 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 15:16:37.401 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 15:16:37.514 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.515 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.607 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.620 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.620 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.621 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.696 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.700 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.704 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.705 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:16:37.706 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.711 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.713 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.727 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.728 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:37.796 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 15:16:37.799 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 15:16:37.800 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:16:38.194 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 15:16:38.199 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 15:16:38.611 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.623 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.706 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.799 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.801 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.807 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.808 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.809 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.813 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.815 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.816 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:16:38.818 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.822 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.894 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 15:16:38.896 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.899 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.904 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:16:38.905 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 15:16:38.912 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 15:16:39.623 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 15:16:39.624 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.702 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.704 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.720 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.802 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.804 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.805 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.806 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.808 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.811 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.818 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 15:16:39.821 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.821 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:16:39.821 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.825 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.825 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.896 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:16:39.912 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 15:16:40.617 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 15:16:40.621 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 15:16:40.789 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 15:16:41.379 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 15:16:41.404 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 15:16:41.931 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:41.997 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.110 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.319 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.403 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.416 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.416 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.417 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.417 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.417 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.422 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.430 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:16:42.602 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.603 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:42.708 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 15:16:42.712 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:43.094 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:16:43.102 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 15:16:43.515 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 15:16:43.601 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 15:16:44.225 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 15:16:44.230 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 15:16:44.627 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:44.904 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:44.920 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.208 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.218 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 15:16:45.220 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.221 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.231 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.233 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.236 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.237 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.237 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.304 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.306 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.315 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:16:45.316 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.316 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:16:45.321 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 15:16:46.424 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 15:16:46.442 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:45 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> [2024/05/06 15:16:46.633 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table test.finish_mark not exists for 12-th check, retry later [2024/05/06 15:16:46.831 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [2024/05/06 15:16:47.019 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:16:47.098 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:16:47.321 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.322 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.328 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.328 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.395 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.410 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 15:16:47.414 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:16:47.414 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.416 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.421 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:16:47.424 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 15:16:47.425 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:16:47.426 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:16:47.429 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:16:47.434 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... pass test: create changefeed v2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:16:48.021 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 15:16:48.026 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c78bc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:1131, start at 2024-05-06 15:16:46.776099821 +0800 CST m=+5.167309062 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.767 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.767 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c78bc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:1131, start at 2024-05-06 15:16:46.776099821 +0800 CST m=+5.167309062 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.767 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.767 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c789c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:1219, start at 2024-05-06 15:16:46.773812245 +0800 CST m=+5.089998882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.780 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:16:48 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60326034.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 41-th check, retry later [2024/05/06 15:16:48.511 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 13-th check, retry later [2024/05/06 15:16:48.612 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:48.619 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:48.824 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:16:48.911 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:16:48.924 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 15:16:48.924 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 15:16:48.930 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 15:16:49.095 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:16:49.105 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:16:49.105 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:16:49.106 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:48 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: remove changefeed v2 [2024/05/06 15:16:49.194 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:16:49.203 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 15:16:49.214 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:49.214 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:49.299 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:49.308 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:49.316 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:16:49.318 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c76100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:1188, start at 2024-05-06 15:16:46.601423803 +0800 CST m=+5.412020941 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c76100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:1188, start at 2024-05-06 15:16:46.601423803 +0800 CST m=+5.412020941 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c7600000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:1254, start at 2024-05-06 15:16:46.604741408 +0800 CST m=+5.356556554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:46.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:46.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:46.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c95cc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:1187, start at 2024-05-06 15:16:48.630565894 +0800 CST m=+5.100322671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:48.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c95cc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:1187, start at 2024-05-06 15:16:48.630565894 +0800 CST m=+5.100322671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:48.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c98140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:1278, start at 2024-05-06 15:16:48.800815248 +0800 CST m=+5.182624042 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:48.809 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c99ac0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:1241, start at 2024-05-06 15:16:48.923329117 +0800 CST m=+5.151019813 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:48.929 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c99ac0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:1241, start at 2024-05-06 15:16:48.923329117 +0800 CST m=+5.151019813 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:48.929 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330c9b280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:1311, start at 2024-05-06 15:16:48.99367908 +0800 CST m=+5.154719016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:49.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:48.970 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:48.970 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 42-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2580.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 14-th check, retry later status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock [2024/05/06 15:16:50.404 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 15:16:50.524 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 15:16:50.622 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:16:50.715 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:16:50.820 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:16:51.024 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.098 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.117 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.125 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.196 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.198 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.198 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 15:16:51.208 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.295 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:16:51.309 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:16:51.311 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:16:51.315 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:16:51.316 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cab7c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:1201, start at 2024-05-06 15:16:50.016860511 +0800 CST m=+5.161667906 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.024 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cab7c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:1201, start at 2024-05-06 15:16:50.016860511 +0800 CST m=+5.161667906 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.024 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.015 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.015 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cab840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:1277, start at 2024-05-06 15:16:50.050682363 +0800 CST m=+5.141942953 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:51 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6781d895-ec6e-4ac0-8fea-e4daf97ea144 {"id":"6781d895-ec6e-4ac0-8fea-e4daf97ea144","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f984a 6781d895-ec6e-4ac0-8fea-e4daf97ea144 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6781d895-ec6e-4ac0-8fea-e4daf97ea144 {"id":"6781d895-ec6e-4ac0-8fea-e4daf97ea144","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f984a 6781d895-ec6e-4ac0-8fea-e4daf97ea144 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6781d895-ec6e-4ac0-8fea-e4daf97ea144 {"id":"6781d895-ec6e-4ac0-8fea-e4daf97ea144","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc21f984a 6781d895-ec6e-4ac0-8fea-e4daf97ea144 /tidb/cdc/default/default/upstream/7365781914866356277 {"id":7365781914866356277,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6077.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:51 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> [2024/05/06 15:16:51.516 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330caf540007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:1251, start at 2024-05-06 15:16:50.268782954 +0800 CST m=+5.163941439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330caf540007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:1251, start at 2024-05-06 15:16:50.268782954 +0800 CST m=+5.163941439 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.276 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cafe80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:1329, start at 2024-05-06 15:16:50.299165546 +0800 CST m=+5.112127537 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:50.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:50.298 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:50.298 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365781914866356277,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:16:51.663845766+08:00","start_ts":449571667705331716,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571667705331716,"checkpoint_ts":449571667705331716,"checkpoint_time":"2024-05-06 15:16:51.498"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449571667369525249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571667369525249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 15:16:51.893 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 15:16:51.906 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:16:51.913 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later run test case test_kill_owner [Mon May 6 15:16:52 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.26352637.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:16:52.010 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:16:52.199 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.208 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [Mon May 6 15:16:51 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.25812583.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 15-th check, retry later [2024/05/06 15:16:52.293 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.297 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.300 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 15:16:52.306 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 15:16:52.307 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 15:16:52.393 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.406 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.407 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:16:52.493 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:16:52.498 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] wait process cdc.test exit for 1-th time... [Mon May 6 15:16:52 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.25682570.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:16:52.500 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:16:52.503 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] wait process cdc.test exit for 2-th time... [Mon May 6 15:16:52 CST 2024] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.25702572.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s [2024/05/06 15:16:53.194 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] wait process cdc.test exit for 3-th time... [Mon May 6 15:16:53 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.25982600.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571667705331716, '"checkpoint_time":' '"2024-05-06' '15:16:51.498",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:16:53 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> [Mon May 6 15:16:53 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.25932595.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later [2024/05/06 15:16:53.616 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table ddl_manager.finish_mark not exists for 44-th check, retry later table test.finish_mark not exists for 16-th check, retry later check diff failed 5-th time, retry later [2024/05/06 15:16:53.897 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:16:53.911 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:16:53.998 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:16:54.202 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:16:54.406 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:16:54.515 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:16:54.623 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 15:16:54.717 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/06 15:16:54.798 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:16:54.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:16:54.832 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:16:54.835 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:16:54.919 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:16:54.942 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:54 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330ceb140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:1152, start at 2024-05-06 15:16:54.107538182 +0800 CST m=+7.282169312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:54.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:54.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:54.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330ceb140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:1152, start at 2024-05-06 15:16:54.107538182 +0800 CST m=+7.282169312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:54.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:54.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:54.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cea3c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:1185, start at 2024-05-06 15:16:54.037594499 +0800 CST m=+7.098295388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:54.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:54.031 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:54.031 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 {"id":"b13779f8-5d45-4e82-bcf8-a4e4611cd5d4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fa53ff b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 /tidb/cdc/default/default/upstream/7365782147717601681 {"id":7365782147717601681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 {"id":"b13779f8-5d45-4e82-bcf8-a4e4611cd5d4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fa53ff b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 /tidb/cdc/default/default/upstream/7365782147717601681 {"id":7365782147717601681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 {"id":"b13779f8-5d45-4e82-bcf8-a4e4611cd5d4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fa53ff b13779f8-5d45-4e82-bcf8-a4e4611cd5d4 /tidb/cdc/default/default/upstream/7365782147717601681 {"id":7365782147717601681,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2618.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2651.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2681.out cli changefeed create --start-ts=449571667369525249 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571667705331716, '"checkpoint_time":' '"2024-05-06' '15:16:51.498",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:16:54 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.26302632.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7021416c-07a2-40f2-814d-4a5b518f3f34 {"id":"7021416c-07a2-40f2-814d-4a5b518f3f34","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fba2f1 7021416c-07a2-40f2-814d-4a5b518f3f34 /tidb/cdc/default/default/upstream/7365782156733957893 {"id":7365782156733957893,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.2659.out cli changefeed create '--sink-uri=mysql://normal:123456@127.0.0.1:3306?transaction-atomicity=none' Create changefeed successfully! ID: d6a91098-7b58-4aa9-a884-b6a9cfdb37ba Info: {"upstream_id":7365782156733957893,"namespace":"default","id":"d6a91098-7b58-4aa9-a884-b6a9cfdb37ba","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306?transaction-atomicity=none","create_time":"2024-05-06T15:16:55.458980944+08:00","start_ts":449571668709081094,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"transaction_atomicity":"none","encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571668709081094,"checkpoint_ts":449571668709081094,"checkpoint_time":"2024-05-06 15:16:55.327"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later Create changefeed successfully! ID: c6a72c52-89de-4b97-94c0-1fc17e986645 Info: {"upstream_id":7365782149811922033,"namespace":"default","id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:16:55.466049261+08:00","start_ts":449571667369525249,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571667369525249,"checkpoint_ts":449571667369525249,"checkpoint_time":"2024-05-06 15:16:50.217"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 45-th check, retry later Create changefeed successfully! ID: 9303e56a-1ac4-4ebe-8d2d-8ffbe4ff81f9 Info: {"upstream_id":7365782147717601681,"namespace":"default","id":"9303e56a-1ac4-4ebe-8d2d-8ffbe4ff81f9","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T15:16:55.68660811+08:00","start_ts":449571668766228490,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571668766228490,"checkpoint_ts":449571668766228490,"checkpoint_time":"2024-05-06 15:16:55.545"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cd9dc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:1198, start at 2024-05-06 15:16:53.005350629 +0800 CST m=+5.230477023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:53.015 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:52.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:52.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cd9dc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:1198, start at 2024-05-06 15:16:53.005350629 +0800 CST m=+5.230477023 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:53.015 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:52.983 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:52.983 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330cdc200006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:1280, start at 2024-05-06 15:16:53.135998011 +0800 CST m=+5.307914208 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:53.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:53.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:53.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Create changefeed successfully! ID: 7f2167c8-9253-4e5b-993f-d00e82fb8ce4 Info: {"upstream_id":7365782157095473129,"namespace":"default","id":"7f2167c8-9253-4e5b-993f-d00e82fb8ce4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:16:55.733043536+08:00","start_ts":449571668787200004,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571668787200004,"checkpoint_ts":449571668787200004,"checkpoint_time":"2024-05-06 15:16:55.625"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:16:56.005 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/06 15:16:56.007 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc2f60d86_5b81_495b_bc98_14177213be25"] check diff successfully [2024/05/06 15:16:56.415 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/06 15:16:56.420 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/06 15:16:56.501 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs230d348f_c956_487b_90b1_70d430d6064d"] [2024/05/06 15:16:56.510 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1e70039-f6dc-4405-8212-55b793e8ef6d {"id":"f1e70039-f6dc-4405-8212-55b793e8ef6d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fee6f0 f1e70039-f6dc-4405-8212-55b793e8ef6d /tidb/cdc/default/default/upstream/7365782161299335213 {"id":7365782161299335213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1e70039-f6dc-4405-8212-55b793e8ef6d {"id":"f1e70039-f6dc-4405-8212-55b793e8ef6d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fee6f0 f1e70039-f6dc-4405-8212-55b793e8ef6d /tidb/cdc/default/default/upstream/7365782161299335213 {"id":7365782161299335213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1e70039-f6dc-4405-8212-55b793e8ef6d {"id":"f1e70039-f6dc-4405-8212-55b793e8ef6d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2fee6f0 f1e70039-f6dc-4405-8212-55b793e8ef6d /tidb/cdc/default/default/upstream/7365782161299335213 {"id":7365782161299335213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 86568eb4-450e-40e7-8230-dcbedb7eca2e Info: {"upstream_id":7365782161299335213,"namespace":"default","id":"86568eb4-450e-40e7-8230-dcbedb7eca2e","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T15:16:56.502685663+08:00","start_ts":449571668155432961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571668155432961,"checkpoint_ts":449571668155432961,"checkpoint_time":"2024-05-06 15:16:53.215"} table test.finish_mark exists check diff successfully [2024/05/06 15:16:56.706 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb9f671c1_33ae_426f_9906_fa08d42fd021"] [2024/05/06 15:16:56.708 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:16:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c2e0a3a-49ae-4138-bc24-297e8bcfa691 {"id":"7c2e0a3a-49ae-4138-bc24-297e8bcfa691","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2ffddf2 7c2e0a3a-49ae-4138-bc24-297e8bcfa691 /tidb/cdc/default/default/upstream/7365782166059883760 {"id":7365782166059883760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c2e0a3a-49ae-4138-bc24-297e8bcfa691 {"id":"7c2e0a3a-49ae-4138-bc24-297e8bcfa691","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2ffddf2 7c2e0a3a-49ae-4138-bc24-297e8bcfa691 /tidb/cdc/default/default/upstream/7365782166059883760 {"id":7365782166059883760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c2e0a3a-49ae-4138-bc24-297e8bcfa691 {"id":"7c2e0a3a-49ae-4138-bc24-297e8bcfa691","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2ffddf2 7c2e0a3a-49ae-4138-bc24-297e8bcfa691 /tidb/cdc/default/default/upstream/7365782166059883760 {"id":7365782166059883760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: e8a723cb-740b-4e54-8d01-b30647b54ec3 Info: {"upstream_id":7365782166059883760,"namespace":"default","id":"e8a723cb-740b-4e54-8d01-b30647b54ec3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:16:56.803488965+08:00","start_ts":449571668232765441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571668232765441,"checkpoint_ts":449571668232765441,"checkpoint_time":"2024-05-06 15:16:53.510"} + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 1-th time... [2024/05/06 15:16:57.024 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa8c756b5_1d78_4f04_9605_641c13c37445"] + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading google.golang.org/grpc v1.62.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 + set +x cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' [2024/05/06 15:16:57.306 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:57.323 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/06 15:16:57.333 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + set +x ***************** properties ***************** "dotransactions"="false" "mysql.port"="4000" "mysql.db"="big_txn" "requestdistribution"="uniform" "threadcount"="1" "recordcount"="5000" "readproportion"="0" "insertproportion"="0" "scanproportion"="0" "workload"="core" "mysql.user"="root" "fieldcount"="100" "operationcount"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" ********************************************** table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later [2024/05/06 15:16:57.412 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsebe6aa09_d78b_4571_bae2_eb1ea67e9247"] [2024/05/06 15:16:57.515 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3b19e0ff_2822_4f5b_af9e_de6220bf7c1c"] [2024/05/06 15:16:57.528 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] wait process cdc.test exit for 2-th time... [2024/05/06 15:16:57.722 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5f7c5918_4545_4362_9d59_7e10351ad073"] [2024/05/06 15:16:57.725 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:57.806 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:57.897 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2424.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2581.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:16:57 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:16:58 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later "is-owner": true, run task successfully [2024/05/06 15:16:58.294 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:58.415 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/05/06 15:16:58.505 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:58.745 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 15:16:58.745 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365781504529130118] [2024/05/06 15:16:58.745 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 15:16:58.745 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 15:16:58.746 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 15:16:58.746 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 15:16:58.747 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 15:16:58.747 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365781504529130118] [2024/05/06 15:16:58.748 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 15:16:58.748 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 15:16:58.748 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 15:16:58.748 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 15:16:58.749 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 15:16:58.749 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 15:16:58.750 +08:00] [INFO] [store_cache.go:510] ["change store resolve state"] [store=2] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:16:58.793 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:58.908 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:58.914 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:59.000 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:59.008 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:59.093 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:59.098 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:59.200 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:16:59.215 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:16:59.219 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:59.223 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:16:59.305 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:16:59.414 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table owner_resign.t1 exists check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571667705331716, '"checkpoint_time":' '"2024-05-06' '15:16:51.498",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 3-th time, retry later [2024/05/06 15:16:59.607 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330d2244001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:1308, start at 2024-05-06 15:16:57.654522255 +0800 CST m=+8.090892744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:57.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:57.617 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:57.617 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330d2244001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:1308, start at 2024-05-06 15:16:57.654522255 +0800 CST m=+8.090892744 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:57.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:57.617 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:57.617 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330d103c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:1375, start at 2024-05-06 15:16:56.503982488 +0800 CST m=+6.876385990 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:18:56.514 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:16:56.514 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:06:56.514 +0800 All versions after safe point can be accessed. (DO NOT EDIT) owner pid: 2640 owner id efd28a9f-42d9-4df2-89d6-19d535a026d6 [Mon May 6 15:16:59 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.27772779.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449571669379907585 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571669379907585 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + set +x + tso='449571669497085953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571669497085953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:16:59 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26162618.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.check1 not exists for 18-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:00.020 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:00.220 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_manager.finish_mark not exists for 47-th check, retry later check diff failed 2-th time, retry later [Mon May 6 15:17:00 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.24532455.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:00.306 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:00.326 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:00.394 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4ab82a75_be6f_43f3_a8df_bacfa92d3c66"] [2024/05/06 15:17:00.405 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:00.500 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:00.504 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:17:00 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> [2024/05/06 15:17:00.608 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:00.617 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:00.630 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:00.707 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:00.717 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:00.802 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:01.023 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:01.299 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:01.599 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:01.725 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:01.795 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:01.800 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:01.816 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:01.818 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table consistent_replicate_storage_file.check1 not exists for 19-th check, retry later [2024/05/06 15:17:01.896 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:02.000 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:02.016 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:02.101 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 48-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully [Mon May 6 15:17:01 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28202822.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:02.200 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:02.201 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:02.216 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:02.226 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2641.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 15:17:02.405 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:02.425 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:02.694 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 30.299846523s"] [2024/05/06 15:17:02.802 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031a1b90 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571667815170068 MinTableBarrierTs:449571670371074060 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/501b3878-4bdb-4191-a511-8cad04aaa355 {"id":"501b3878-4bdb-4191-a511-8cad04aaa355","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8145e 501b3878-4bdb-4191-a511-8cad04aaa355 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571667815170068,"min-table-barrier-ts":449571670371074060,"admin-job-type":0} /tidb/cdc/default/default/task/position/501b3878-4bdb-4191-a511-8cad04aaa355/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/efd28a9f-42d9-4df2-89d6-19d535a026d6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031a1b90 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571667815170068 MinTableBarrierTs:449571670371074060 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/501b3878-4bdb-4191-a511-8cad04aaa355 {"id":"501b3878-4bdb-4191-a511-8cad04aaa355","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8145e 501b3878-4bdb-4191-a511-8cad04aaa355 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571667815170068,"min-table-barrier-ts":449571670371074060,"admin-job-type":0} /tidb/cdc/default/default/task/position/501b3878-4bdb-4191-a511-8cad04aaa355/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/efd28a9f-42d9-4df2-89d6-19d535a026d6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 + grep -q 'failed to get info:' {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0031a1b90 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571667815170068 MinTableBarrierTs:449571670371074060 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/501b3878-4bdb-4191-a511-8cad04aaa355 {"id":"501b3878-4bdb-4191-a511-8cad04aaa355","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/efd28a9f-42d9-4df2-89d6-19d535a026d6 {"id":"efd28a9f-42d9-4df2-89d6-19d535a026d6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81413 efd28a9f-42d9-4df2-89d6-19d535a026d6 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8145e 501b3878-4bdb-4191-a511-8cad04aaa355 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571667815170068,"min-table-barrier-ts":449571670371074060,"admin-job-type":0} /tidb/cdc/default/default/task/position/501b3878-4bdb-4191-a511-8cad04aaa355/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/efd28a9f-42d9-4df2-89d6-19d535a026d6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "efd28a9f-42d9-4df2-89d6-19d535a026d6" | grep -v cluster_id | grep id table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66585182-2f7a-48ce-a248-d4cad5757604 {"id":"66585182-2f7a-48ce-a248-d4cad5757604","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc30a4cf5 66585182-2f7a-48ce-a248-d4cad5757604 /tidb/cdc/default/default/upstream/7365782175197267748 {"id":7365782175197267748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66585182-2f7a-48ce-a248-d4cad5757604 {"id":"66585182-2f7a-48ce-a248-d4cad5757604","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc30a4cf5 66585182-2f7a-48ce-a248-d4cad5757604 /tidb/cdc/default/default/upstream/7365782175197267748 {"id":7365782175197267748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66585182-2f7a-48ce-a248-d4cad5757604 {"id":"66585182-2f7a-48ce-a248-d4cad5757604","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc30a4cf5 66585182-2f7a-48ce-a248-d4cad5757604 /tidb/cdc/default/default/upstream/7365782175197267748 {"id":7365782175197267748,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2673.out cli changefeed create --start-ts=449571669497085953 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/06 15:17:03.220 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:03.294 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:03.310 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:03.323 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:03.328 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Create changefeed successfully! ID: 3a0ff234-b476-42c4-b1ca-12156b3d6d7c Info: {"upstream_id":7365782175197267748,"namespace":"default","id":"3a0ff234-b476-42c4-b1ca-12156b3d6d7c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:03.26295656+08:00","start_ts":449571669497085953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571669497085953,"checkpoint_ts":449571669497085953,"checkpoint_time":"2024-05-06 15:16:58.333"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:03.501 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:03.530 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:03.603 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:03.618 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4144b86-4655-42f9-8cb8-aea91ca07d58 {"id":"e4144b86-4655-42f9-8cb8-aea91ca07d58","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3060100 e4144b86-4655-42f9-8cb8-aea91ca07d58 /tidb/cdc/default/default/upstream/7365782168095015858 {"id":7365782168095015858,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4144b86-4655-42f9-8cb8-aea91ca07d58 {"id":"e4144b86-4655-42f9-8cb8-aea91ca07d58","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3060100 e4144b86-4655-42f9-8cb8-aea91ca07d58 /tidb/cdc/default/default/upstream/7365782168095015858 {"id":7365782168095015858,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4144b86-4655-42f9-8cb8-aea91ca07d58 {"id":"e4144b86-4655-42f9-8cb8-aea91ca07d58","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3060100 e4144b86-4655-42f9-8cb8-aea91ca07d58 /tidb/cdc/default/default/upstream/7365782168095015858 {"id":7365782168095015858,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2494.out cli changefeed create --start-ts=449571669379907585 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail table consistent_replicate_storage_file.check1 not exists for 20-th check, retry later + set +x + tso='449571670541729793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571670541729793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:17:03 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.26782680.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:03.721 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:03.823 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:03.823 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:03.900 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:03.917 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:03.930 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7365782168095015858,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:17:03.96110455+08:00","start_ts":449571669379907585,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571669379907585,"checkpoint_ts":449571669379907585,"checkpoint_time":"2024-05-06 15:16:57.886"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:17:03 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 49-th check, retry later [2024/05/06 15:17:04.103 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:04.424 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] "cluster-id": "default" "id": "501b3878-4bdb-4191-a511-8cad04aaa355", "cluster-id": "default" run task successfully [2024/05/06 15:17:04.519 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:04.519 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:04.717 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:04 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {UpstreamID:7365782157095473129 Namespace:default ID:7f2167c8-9253-4e5b-993f-d00e82fb8ce4 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.733043536 +0800 CST StartTs:449571668787200004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00328ae10 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668813414402} {CheckpointTs:449571670937042960 MinTableBarrierTs:449571670937042960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 {"id":"4f6e12a1-5b02-4397-bbea-17e6c0abe3b7","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa666 4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 /tidb/cdc/default/default/changefeed/info/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"upstream-id":7365782157095473129,"namespace":"default","changefeed-id":"7f2167c8-9253-4e5b-993f-d00e82fb8ce4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.733043536+08:00","start-ts":449571668787200004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668813414402} /tidb/cdc/default/default/changefeed/status/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":449571670937042960,"min-table-barrier-ts":449571670937042960,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/52944ec3-aeae-47b5-b375-7aca8cef9634/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {UpstreamID:7365782157095473129 Namespace:default ID:7f2167c8-9253-4e5b-993f-d00e82fb8ce4 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.733043536 +0800 CST StartTs:449571668787200004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00328ae10 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668813414402} {CheckpointTs:449571670937042960 MinTableBarrierTs:449571670937042960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 {"id":"4f6e12a1-5b02-4397-bbea-17e6c0abe3b7","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa666 4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 /tidb/cdc/default/default/changefeed/info/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"upstream-id":7365782157095473129,"namespace":"default","changefeed-id":"7f2167c8-9253-4e5b-993f-d00e82fb8ce4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.733043536+08:00","start-ts":449571668787200004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668813414402} /tidb/cdc/default/default/changefeed/status/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":449571670937042960,"min-table-barrier-ts":449571670937042960,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/52944ec3-aeae-47b5-b375-7aca8cef9634/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {UpstreamID:7365782157095473129 Namespace:default ID:7f2167c8-9253-4e5b-993f-d00e82fb8ce4 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.733043536 +0800 CST StartTs:449571668787200004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00328ae10 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668813414402} {CheckpointTs:449571670937042960 MinTableBarrierTs:449571670937042960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 {"id":"4f6e12a1-5b02-4397-bbea-17e6c0abe3b7","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/52944ec3-aeae-47b5-b375-7aca8cef9634 {"id":"52944ec3-aeae-47b5-b375-7aca8cef9634","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa5f3 52944ec3-aeae-47b5-b375-7aca8cef9634 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2faa666 4f6e12a1-5b02-4397-bbea-17e6c0abe3b7 /tidb/cdc/default/default/changefeed/info/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"upstream-id":7365782157095473129,"namespace":"default","changefeed-id":"7f2167c8-9253-4e5b-993f-d00e82fb8ce4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.733043536+08:00","start-ts":449571668787200004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668813414402} /tidb/cdc/default/default/changefeed/status/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":449571670937042960,"min-table-barrier-ts":449571670937042960,"admin-job-type":0} /tidb/cdc/default/default/task/position/4f6e12a1-5b02-4397-bbea-17e6c0abe3b7/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/52944ec3-aeae-47b5-b375-7aca8cef9634/7f2167c8-9253-4e5b-993f-d00e82fb8ce4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782157095473129 {"id":7365782157095473129,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' + set +x [2024/05/06 15:17:04.893 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later [2024/05/06 15:17:05.125 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:05.208 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:05.212 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:05.233 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:05.432 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table consistent_replicate_storage_file.check1 not exists for 21-th check, retry later capture_id: 501b3878-4bdb-4191-a511-8cad04aaa355 [2024/05/06 15:17:05.926 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:05.933 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:06.033 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 2640 exit for 1-th time... table ddl_manager.finish_mark not exists for 50-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7365782168095015858, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:17:03.977438416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365782168095015858, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:17:03.977438416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365782168095015858, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T15:17:03.977438416+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7365782168095015858, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T15:17:03.977438416+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365782168095015858, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T15:17:03.977438416+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571667705331716, '"checkpoint_time":' '"2024-05-06' '15:16:51.498",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 4-th time, retry later [2024/05/06 15:17:06.114 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:06.209 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:06.319 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:06.430 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:06.526 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:06.529 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:06.531 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff failed 5-th time, retry later wait process 2640 exit for 2-th time... table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later [2024/05/06 15:17:06.594 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:06.719 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:06.802 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later [2024/05/06 15:17:06.824 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:06.832 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:07.000 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50c655a5-f9a3-439b-81ea-98a7045ccdd4 {"id":"50c655a5-f9a3-439b-81ea-98a7045ccdd4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3152df7 50c655a5-f9a3-439b-81ea-98a7045ccdd4 /tidb/cdc/default/default/upstream/7365782183345268435 {"id":7365782183345268435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50c655a5-f9a3-439b-81ea-98a7045ccdd4 {"id":"50c655a5-f9a3-439b-81ea-98a7045ccdd4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3152df7 50c655a5-f9a3-439b-81ea-98a7045ccdd4 /tidb/cdc/default/default/upstream/7365782183345268435 {"id":7365782183345268435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50c655a5-f9a3-439b-81ea-98a7045ccdd4 {"id":"50c655a5-f9a3-439b-81ea-98a7045ccdd4","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3152df7 50c655a5-f9a3-439b-81ea-98a7045ccdd4 /tidb/cdc/default/default/upstream/7365782183345268435 {"id":7365782183345268435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1414 --:--:-- --:--:-- --:--:-- 1420 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2640) - No such process wait process 2640 exit for 3-th time... process 2640 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 501b3878-4bdb-4191-a511-8cad04aaa355 -A1 | grep '"is-owner": true' [2024/05/06 15:17:07.296 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:07.316 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:17:06 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> INSERT - Takes(s): 10.0, Count: 3244, OPS: 324.6, Avg(us): 2813, Min(us): 1263, Max(us): 84172, 95th(us): 3000, 99th(us): 70000 table consistent_replicate_storage_file.check1 not exists for 22-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/06 15:17:07.494 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:07.518 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 2-th time... [2024/05/06 15:17:07.717 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:07.812 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later [2024/05/06 15:17:07.907 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:07.928 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:07.931 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:08.007 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:08.017 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:08.134 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:08.331 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:08.332 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff successfully "is-owner": true, run task successfully test_kill_owner: pass check diff successfully [2024/05/06 15:17:08.414 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:08.429 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:08.520 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:08.601 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:17:08.753 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 15:17:08.753 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 15:17:08.753 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 15:17:08.753 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 15:17:08.754 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 15:17:08.754 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] wait process cdc.test exit for 1-th time... [2024/05/06 15:17:08.808 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:17:08 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> [2024/05/06 15:17:09.099 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] wait process cdc.test exit for 1-th time... [2024/05/06 15:17:09.295 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:09.305 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:09.392 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:09.395 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:09.398 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:09.408 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:09.411 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 2-th time... [2024/05/06 15:17:09.499 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:09.615 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:09.619 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 wait process cdc.test exit for 2-th time... [2024/05/06 15:17:09.026 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:17:09.027 +08:00] [INFO] [case.go:445] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:17:09.027 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq0] [2024/05/06 15:17:09.554 +08:00] [INFO] [case.go:365] ["insert 100 takes 9.617133ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 15:17:09.555 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq1] [2024/05/06 15:17:09.693 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:09.698 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:09.700 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:09.709 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:09.727 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:09.822 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:17:09 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:09 CST 2024] <<<<<< run test case tiflash success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_owner [Mon May 6 15:17:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.29452947.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330dd1b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:9009, start at 2024-05-06 15:17:08.862907148 +0800 CST m=+5.079523303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:08.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:08.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:08.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330dd1b40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:9009, start at 2024-05-06 15:17:08.862907148 +0800 CST m=+5.079523303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:08.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:08.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:08.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330dd2740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wb6ft, pid:9085, start at 2024-05-06 15:17:08.937687612 +0800 CST m=+5.098458198 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:08.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:08.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:08.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure table ddl_manager.finish_mark not exists for 52-th check, retry later table consistent_replicate_storage_file.check1 not exists for 23-th check, retry later [2024/05/06 15:17:10.004 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:10.102 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... [2024/05/06 15:17:10.074 +08:00] [INFO] [case.go:365] ["insert 100 takes 8.977799ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 15:17:10.075 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq2] check diff failed 1-th time, retry later [2024/05/06 15:17:10.297 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:10.306 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:10.397 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:10.416 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:10.593 +08:00] [INFO] [case.go:365] ["insert 100 takes 8.265937ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 15:17:10.593 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq3] [2024/05/06 15:17:10.505 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:10.520 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:10.828 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:10.904 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:10.925 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:11.116 +08:00] [INFO] [case.go:365] ["insert 100 takes 12.376852ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 15:17:11.117 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts_seq4] [2024/05/06 15:17:11.007 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:11.009 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:11.018 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:11.027 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:11.206 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:11.313 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:11.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:11.414 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff failed 3-th time, retry later [2024/05/06 15:17:11.603 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:11.612 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:11.619 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:11.637 +08:00] [INFO] [case.go:365] ["insert 100 takes 8.679215ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2024/05/06 15:17:11.637 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts0] table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 53-th check, retry later [2024/05/06 15:17:11.804 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10464.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 15:17:12.100 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:12.169 +08:00] [INFO] [case.go:365] ["insert 100 takes 22.092206ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 15:17:12.169 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts1] check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:12.307 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:12.399 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:12.402 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:12.406 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:12.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:12.611 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:12.622 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 [2024/05/06 15:17:12.691 +08:00] [INFO] [case.go:365] ["insert 100 takes 11.359936ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 15:17:12.691 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts2] [2024/05/06 15:17:12.735 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:12.825 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:12.826 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:12.916 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:12.929 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:17:12 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1abd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571670908469250 MinTableBarrierTs:449571673267765249 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571670908469250,"min-table-barrier-ts":449571673267765249,"admin-job-type":0} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1abd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571670908469250 MinTableBarrierTs:449571673267765249 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571670908469250,"min-table-barrier-ts":449571673267765249,"admin-job-type":0} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.+ grep -q 'failed to get info:' 0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1abd0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571670908469250 MinTableBarrierTs:449571673267765249 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571670908469250, checkpointTs: 449571670908469250, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571670908469250,"min-table-barrier-ts":449571673267765249,"admin-job-type":0} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff failed 4-th time, retry later [2024/05/06 15:17:13.212 +08:00] [INFO] [case.go:365] ["insert 100 takes 10.673433ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 15:17:13.213 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts3] [2024/05/06 15:17:13.296 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later [2024/05/06 15:17:13.609 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:13.697 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:13.704 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:13.794 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:13.735 +08:00] [INFO] [case.go:365] ["insert 100 takes 10.733947ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 15:17:13.735 +08:00] [INFO] [case.go:383] ["drop tables"] [table=accounts4] Run finished, takes 15.98800883s INSERT - Takes(s): 16.0, Count: 5000, OPS: 312.9, Avg(us): 2905, Min(us): 1263, Max(us): 88789, 95th(us): 3000, 99th(us): 69000 table big_txn.usertable exists + set +x + tso='449571673183354881 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571673183354881 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:17:13 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1050610508.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449571667705331716, "checkpoint_time": "2024-05-06 15:16:51.498", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449571667705331716, '"checkpoint_time":' '"2024-05-06' '15:16:51.498",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:14.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:14.134 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:14.201 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:14.219 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:14.300 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:14.315 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table ddl_manager.finish_mark not exists for 54-th check, retry later [2024/05/06 15:17:14.259 +08:00] [INFO] [case.go:365] ["insert 100 takes 10.872345ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2024/05/06 15:17:14.404 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:14.605 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:14.612 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:14.622 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] "is-owner": true, run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:17:14.716 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 5-th time, retry later check diff failed 2-th time, retry later table test.t2 not exists for 1-th check, retry later [2024/05/06 15:17:15.301 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:15.308 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:15.318 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:15.422 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:15.519 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:15.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:15.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:15.710 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 15:17:15.826 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:15.834 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:15.906 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa44de1b4_3b6a_4358_bd99_c75d9f3ad891"] [2024/05/06 15:17:15.923 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs58c43951_d553_4d28_bf7a_071910f332ab"] owner pid: 2950 owner id 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 [Mon May 6 15:17:15 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30553057.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:16.195 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs86481989_1362_477d_bff7_4babdcae11be"] [2024/05/06 15:17:16.196 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsaff1cb08_f7d7_4bfe_8a02_ee01a8ebac38"] [2024/05/06 15:17:16.200 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table ddl_manager.finish_mark not exists for 55-th check, retry later [2024/05/06 15:17:16.321 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs759c83dd_a568_48b6_b68d_fe5407176630"] [2024/05/06 15:17:16.398 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs13e6af5c_9f0d_40e2_866d_5fa0d5baf62a"] table test.t2 not exists for 2-th check, retry later check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e37e80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:7163, start at 2024-05-06 15:17:15.387445034 +0800 CST m=+5.156736708 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:15.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:15.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:15.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e37e80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:7163, start at 2024-05-06 15:17:15.387445034 +0800 CST m=+5.156736708 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:15.395 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:15.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:15.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e388c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:7228, start at 2024-05-06 15:17:15.428352789 +0800 CST m=+5.149255106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:15.437 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:15.427 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:15.427 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:17.003 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:16.771 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a2b0c3-9e39-4d8c-b89b-802f50d781a9 {"id":"17a2b0c3-9e39-4d8c-b89b-802f50d781a9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc34cc2f3 17a2b0c3-9e39-4d8c-b89b-802f50d781a9 /tidb/cdc/default/default/upstream/7365782243603685845 {"id":7365782243603685845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a2b0c3-9e39-4d8c-b89b-802f50d781a9 {"id":"17a2b0c3-9e39-4d8c-b89b-802f50d781a9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc34cc2f3 17a2b0c3-9e39-4d8c-b89b-802f50d781a9 /tidb/cdc/default/default/upstream/7365782243603685845 {"id":7365782243603685845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17a2b0c3-9e39-4d8c-b89b-802f50d781a9 {"id":"17a2b0c3-9e39-4d8c-b89b-802f50d781a9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc34cc2f3 17a2b0c3-9e39-4d8c-b89b-802f50d781a9 /tidb/cdc/default/default/upstream/7365782243603685845 {"id":7365782243603685845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10556.out cli changefeed create --start-ts=449571673183354881 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [2024/05/06 15:17:17.098 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:17.101 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Create changefeed successfully! ID: daa067f5-2115-49b4-8c0f-d9ebedbf9447 Info: {"upstream_id":7365782243603685845,"namespace":"default","id":"daa067f5-2115-49b4-8c0f-d9ebedbf9447","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:17.28776895+08:00","start_ts":449571673183354881,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571673183354881,"checkpoint_ts":449571673183354881,"checkpoint_time":"2024-05-06 15:17:12.395"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:17.305 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:17.306 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:17.396 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:17.409 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:17.412 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:17.501 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later check diff failed 6-th time, retry later [2024/05/06 15:17:17.595 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:17.600 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:17.607 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6da189aa_06ed_40b3_98a5_5ff135e3d8e0"] [2024/05/06 15:17:17.701 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) table ddl_manager.finish_mark not exists for 56-th check, retry later [2024/05/06 15:17:18.323 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:18.328 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.t2 not exists for 3-th check, retry later [2024/05/06 15:17:18.399 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:18.498 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:18.499 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:18.503 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:18.508 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:18.513 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:18.610 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:18.697 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:18.698 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:18.706 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:18.710 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:18.712 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:18.784 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033b3e60 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571674578485251 MinTableBarrierTs:449571674578485251 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59c695d1-337a-42b9-9cee-06230017f68c {"id":"59c695d1-337a-42b9-9cee-06230017f68c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814d0 59c695d1-337a-42b9-9cee-06230017f68c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571674578485251,"min-table-barrier-ts":449571674578485251,"admin-job-type":0} /tidb/cdc/default/default/task/position/59c695d1-337a-42b9-9cee-06230017f68c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033b3e60 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571674578485251 MinTableBarrierTs:449571674578485251 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59c695d1-337a-42b9-9cee-06230017f68c {"id":"59c695d1-337a-42b9-9cee-06230017f68c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814d0 59c695d1-337a-42b9-9cee-06230017f68c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571674578485251,"min-table-barrier-ts":449571674578485251,"admin-job-type":0} /tidb/cdc/default/default/task/position/59c695d1-337a-42b9-9cee-06230017f68c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033b3e60 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571674578485251 MinTableBarrierTs:449571674578485251 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59c695d1-337a-42b9-9cee-06230017f68c {"id":"59c695d1-337a-42b9-9cee-06230017f68c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/706bbc82-20eb-42b8-aeef-e3e6436e7fc1 {"id":"706bbc82-20eb-42b8-aeef-e3e6436e7fc1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814a0 706bbc82-20eb-42b8-aeef-e3e6436e7fc1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f814d0 59c695d1-337a-42b9-9cee-06230017f68c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571674578485251,"min-table-barrier-ts":449571674578485251,"admin-job-type":0} /tidb/cdc/default/default/task/position/59c695d1-337a-42b9-9cee-06230017f68c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/706bbc82-20eb-42b8-aeef-e3e6436e7fc1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "706bbc82-20eb-42b8-aeef-e3e6436e7fc1" | grep id [Mon May 6 15:17:19 CST 2024] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.84818483.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:19.408 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:19.409 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] check diff successfully check diff failed 1-th time, retry later [2024/05/06 15:17:19.496 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:19.512 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:19.513 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:19.517 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:19.520 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:19.522 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:19.602 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:19.615 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:19.616 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:19.619 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:19.624 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:19.625 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later "cluster-id": "default" "id": "59c695d1-337a-42b9-9cee-06230017f68c", "cluster-id": "default" run task successfully check diff successfully table test.t2 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:20.304 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs05de73f4_a30b_42fb_94ee_de59fc992560"] [2024/05/06 15:17:20.501 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.504 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.596 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.611 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.612 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.694 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.705 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.709 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.716 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.799 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:20.801 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.805 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:20.819 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:20.823 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:20.797 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] wait process cdc.test exit for 1-th time... check diff failed 5-th time, retry later capture_id: 59c695d1-337a-42b9-9cee-06230017f68c ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '59c695d1-337a-42b9-9cee-06230017f68c' wait process cdc.test exit for 2-th time... [2024/05/06 15:17:21.492 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:21.503 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] 59c695d1-337a-42b9-9cee-06230017f68c run task successfully test_hang_up_owner: pass table consistent_replicate_storage_file.t2 exists [2024/05/06 15:17:21.795 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:21.796 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:21.812 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:21.816 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:21.822 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:21.894 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:21.908 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:21.922 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:21.923 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:22.015 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:22.021 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table ddl_manager.finish_mark not exists for 58-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:21 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> check diff successfully [2024/05/06 15:17:22.021 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:22.025 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:22.029 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... table test.t2 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e9274000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:20813, start at 2024-05-06 15:17:21.198618529 +0800 CST m=+5.224271850 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:21.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:21.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:21.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:22.409 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:22.410 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/319c1f54-46d5-401d-9346-31fe2b9c4a48 {"id":"319c1f54-46d5-401d-9346-31fe2b9c4a48","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc367dffe 319c1f54-46d5-401d-9346-31fe2b9c4a48 /tidb/cdc/default/default/upstream/7365782271637626663 {"id":7365782271637626663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/319c1f54-46d5-401d-9346-31fe2b9c4a48 {"id":"319c1f54-46d5-401d-9346-31fe2b9c4a48","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc367dffe 319c1f54-46d5-401d-9346-31fe2b9c4a48 /tidb/cdc/default/default/upstream/7365782271637626663 {"id":7365782271637626663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/319c1f54-46d5-401d-9346-31fe2b9c4a48 {"id":"319c1f54-46d5-401d-9346-31fe2b9c4a48","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc367dffe 319c1f54-46d5-401d-9346-31fe2b9c4a48 /tidb/cdc/default/default/upstream/7365782271637626663 {"id":7365782271637626663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7365782271637626663,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2024-05-06T15:17:22.395781746+08:00","start_ts":449571675789590530,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571675789590530,"checkpoint_ts":449571675789590530,"checkpoint_time":"2024-05-06 15:17:22.337"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + info='{ "upstream_id": 7365782271637626663, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449571675789590530, "checkpoint_time": "2024-05-06 15:17:22.337", "error": null }' + echo '{ "upstream_id": 7365782271637626663, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449571675789590530, "checkpoint_time": "2024-05-06 15:17:22.337", "error": null }' { "upstream_id": 7365782271637626663, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 449571675789590530, "checkpoint_time": "2024-05-06 15:17:22.337", "error": null } ++ echo '{' '"upstream_id":' 7365782271637626663, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449571675789590530, '"checkpoint_time":' '"2024-05-06' '15:17:22.337",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365782271637626663, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 449571675789590530, '"checkpoint_time":' '"2024-05-06' '15:17:22.337",' '"error":' null '}' ++ jq -r .error.message wait process cdc.test exit for 2-th time... check diff successfully [Pipeline] // cache wait process cdc.test exit for 1-th time... + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/gin-gonic/gin v1.8.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... [2024/05/06 15:17:22.916 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:22.999 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:23.005 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:23.013 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_expire_owner [Mon May 6 15:17:23 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.31943196.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:23.098 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:23.102 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:23.109 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:23.118 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:23.194 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:23.215 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:23.215 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:23.301 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:23.302 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:23.307 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:22.811 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 check diff failed 6-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:23 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> wait process cdc.test exit for 3-th time... go: downloading blainsmith.com/go/seahash v1.2.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/google/btree v1.1.2 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 [2024/05/06 15:17:23.611 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:23.612 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:23.819 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:23.823 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 table ddl_manager.finish_mark not exists for 59-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:23 CST 2024] <<<<<< run test case owner_resign success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] } [2024/05/06 15:17:23.910 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:24.004 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:24.102 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:24.104 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:17:24 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1139011392.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s table test.t2 not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e9274000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:20813, start at 2024-05-06 15:17:21.198618529 +0800 CST m=+5.224271850 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:21.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:21.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:21.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330e93e4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:20896, start at 2024-05-06 15:17:21.28942277 +0800 CST m=+5.265618740 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:21.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:21.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:21.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:24.195 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:24.298 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:24.309 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:24.404 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:24.408 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6339.out cli changefeed pause -c changefeed-initialize-error Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:17:24.512 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:24.521 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:24.523 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 PASS [2024/05/06 15:17:24.804 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:24.807 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] go: downloading golang.org/x/text v0.14.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/imdario/mergo v0.3.16 coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:24.998 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:25.000 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:25.015 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:25.101 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:25.102 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:25.108 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:25.111 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 [2024/05/06 15:17:24.824 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/06 15:17:25.401 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:25.402 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 [2024/05/06 15:17:25.493 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:25.504 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-playground/validator/v10 v10.11.1 go: downloading github.com/pelletier/go-toml/v2 v2.0.5 go: downloading github.com/ugorji/go/codec v1.2.7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 [Pipeline] // cache [Pipeline] // dir go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading github.com/AthenZ/athenz v1.10.39 check diff failed 7-th time, retry later [2024/05/06 15:17:25.997 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:26.001 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:26.006 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:26.119 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:26.202 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": { "time": "2024-05-06T15:16:52.036257545+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' '{' '"time":' '"2024-05-06T15:16:52.036257545+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6404.out cli changefeed resume -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 60-th check, retry later [2024/05/06 15:17:26.316 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:26.319 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:26.400 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:26.404 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:26.409 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:26.413 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:26.418 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042894d0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571675364917251 MinTableBarrierTs:449571676675637252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec2ec576-27d3-4710-990c-a812fb3706bb {"id":"ec2ec576-27d3-4710-990c-a812fb3706bb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81509 ec2ec576-27d3-4710-990c-a812fb3706bb /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571675364917251,"min-table-barrier-ts":449571676675637252,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec2ec576-27d3-4710-990c-a812fb3706bb/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042894d0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571675364917251 MinTableBarrierTs:449571676675637252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec2ec576-27d3-4710-990c-a812fb3706bb {"id":"ec2ec576-27d3-4710-990c-a812fb3706bb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81509 ec2ec576-27d3-4710-990c-a812fb3706bb /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571675364917251,"min-table-barrier-ts":449571676675637252,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec2ec576-27d3-4710-990c-a812fb3706bb/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042894d0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571675364917251 MinTableBarrierTs:449571676675637252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571675364917251, checkpointTs: 449571675364917251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec2ec576-27d3-4710-990c-a812fb3706bb {"id":"ec2ec576-27d3-4710-990c-a812fb3706bb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81509 ec2ec576-27d3-4710-990c-a812fb3706bb /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571675364917251,"min-table-barrier-ts":449571676675637252,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec2ec576-27d3-4710-990c-a812fb3706bb/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.+ grep -q 'etcd info' 0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... [2024/05/06 15:17:26.513 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:26.514 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:26.514 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:26.516 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Pipeline] } [Pipeline] } table test.t2 not exists for 7-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.22234.out cli tso query --pd=http://127.0.0.1:2379 Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:17:26.836 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] // dir [2024/05/06 15:17:27.007 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:27.008 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:27.096 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [Pipeline] // withCredentials + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/73723af2-c490-450e-a190-817e0e483b07 {UpstreamID:7365781997393306027 Namespace:default ID:73723af2-c490-450e-a190-817e0e483b07 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:20.157199883 +0800 CST StartTs:449571659471388674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004112900 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571659484233732} {CheckpointTs:449571676707356674 MinTableBarrierTs:449571676707356676 AdminJobType:noop} span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e8d1b7-a40c-418a-bf28-6c1a227bf810 {"id":"48e8d1b7-a40c-418a-bf28-6c1a227bf810","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d4822 48e8d1b7-a40c-418a-bf28-6c1a227bf810 /tidb/cdc/default/default/changefeed/info/73723af2-c490-450e-a190-817e0e483b07 {"upstream-id":7365781997393306027,"namespace":"default","changefeed-id":"73723af2-c490-450e-a190-817e0e483b07","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:20.157199883+08:00","start-ts":449571659471388674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571659484233732} /tidb/cdc/default/default/changefeed/status/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":449571676707356674,"min-table-barrier-ts":449571676969500676,"admin-job-type":0} /tidb/cdc/default/default/task/position/48e8d1b7-a40c-418a-bf28-6c1a227bf810/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/73723af2-c490-450e-a190-817e0e483b07 {UpstreamID:7365781997393306027 Namespace:default ID:73723af2-c490-450e-a190-817e0e483b07 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:20.157199883 +0800 CST StartTs:449571659471388674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004112900 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571659484233732} {CheckpointTs:449571676707356674 MinTableBarrierTs:449571676707356676 AdminJobType:noop} span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e8d1b7-a40c-418a-bf28-6c1a227bf810 {"id":"48e8d1b7-a40c-418a-bf28-6c1a227bf810","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d4822 48e8d1b7-a40c-418a-bf28-6c1a227bf810 /tidb/cdc/default/default/changefeed/info/73723af2-c490-450e-a190-817e0e483b07 {"upstream-id":7365781997393306027,"namespace":"default","changefeed-id":"73723af2-c490-450e-a190-817e0e483b07","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:20.157199883+08:00","start-ts":449571659471388674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571659484233732} /tidb/cdc/default/default/changefeed/status/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":449571676707356674,"min-table-barrier-ts":449571676969500676,"admin-job-type":0} /tidb/cdc/default/default/task/position/48e8d1b7-a40c-418a-bf28-6c1a227bf810/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/73723af2-c490-450e-a190-817e0e483b07 {UpstreamID:7365781997393306027 Namespace:default ID:73723af2-c490-450e-a190-817e0e483b07 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:20.157199883 +0800 CST StartTs:449571659471388674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004112900 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571659484233732} {CheckpointTs:449571676707356674 MinTableBarrierTs:449571676707356676 AdminJobType:noop} span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571676707356676, checkpointTs: 449571676707356674, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571676707356675, checkpointTs: 449571676707356674, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571676707356674, checkpointTs: 449571676707356674, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e8d1b7-a40c-418a-bf28-6c1a227bf810 {"id":"48e8d1b7-a40c-418a-bf28-6c1a227bf810","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc26d4822 48e8d1b7-a40c-418a-bf28-6c1a227bf810 /tidb/cdc/default/default/changefeed/info/73723af2-c490-450e-a190-817e0e483b07 {"upstream-id":7365781997393306027,"namespace":"default","changefeed-id":"73723af2-c490-450e-a190-817e0e483b07","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:20.157199883+08:00","start-ts":449571659471388674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571659484233732} /tidb/cdc/default/default/changefeed/status/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":449571676707356674,"min-table-barrier-ts":449571676969500676,"admin-job-type":0} /tidb/cdc/default/default/task/position/48e8d1b7-a40c-418a-bf28-6c1a227bf810/73723af2-c490-450e-a190-817e0e483b07 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365781997393306027 {"id":7365781997393306027,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/go-playground/universal-translator v0.18.0 go: downloading github.com/leodido/go-urn v1.2.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 [2024/05/06 15:17:27.301 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:27.396 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:27.504 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:27.519 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.523 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/go-playground/locales v0.14.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/json-iterator/go v1.1.12 [2024/05/06 15:17:27.598 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.602 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:27.609 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:27.609 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:27.622 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.625 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.699 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:27.702 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 8-th time, retry later "is-owner": true, run task successfully go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c [Pipeline] } [Pipeline] } [2024/05/06 15:17:27.904 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.904 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:27.905 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:28.027 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 [2024/05/06 15:17:28.105 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table ddl_manager.finish_mark not exists for 61-th check, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 [Pipeline] // timeout [Pipeline] // withCredentials + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null }' + echo '{ "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null }' { "upstream_id": 7365781914866356277, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449571676185690113, "checkpoint_time": "2024-05-06 15:17:23.848", "error": null } ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365781914866356277, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449571676185690113, '"checkpoint_time":' '"2024-05-06' '15:17:23.848",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6462.out cli changefeed remove -c changefeed-initialize-error [2024/05/06 15:17:28.402 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.501 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [Pipeline] } [Pipeline] } table test.t2 not exists for 8-th check, retry later + set +x + tso='449571676993880066 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571676993880066 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:17:28 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2226422266.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:28.603 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.609 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:28.617 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.701 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.702 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.708 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:28.710 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:28.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:28.797 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout [Pipeline] // stage [2024/05/06 15:17:28.848 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/06 15:17:28.915 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:28.919 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:28.920 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:29.025 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Pipeline] } [Pipeline] } owner pid: 3199 owner id ec2ec576-27d3-4710-990c-a812fb3706bb process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'ec2ec576-27d3-4710-990c-a812fb3706bb' /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81509 run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Pipeline] // stage [2024/05/06 15:17:29.100 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:29.297 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:29.324 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b [Pipeline] // container [Pipeline] } [Pipeline] } Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449571676185690113 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:29.411 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:29.498 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:29.500 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:29.507 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:29.511 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [Pipeline] // container [Pipeline] // withEnv [Pipeline] } [Pipeline] } check diff failed 9-th time, retry later [Pipeline] // withEnv [2024/05/06 15:17:29.602 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:29.695 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [Pipeline] // node [Pipeline] } [Pipeline] } [2024/05/06 15:17:29.914 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:29.919 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:29.995 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:30.013 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:30.023 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [Pipeline] // node [Pipeline] // podTemplate table ddl_manager.finish_mark not exists for 62-th check, retry later [Pipeline] } [Pipeline] } Verifying downstream PD is started... go: downloading github.com/jmespath/go-jmespath v0.4.0 [2024/05/06 15:17:30.322 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] } [Pipeline] } go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 [2024/05/06 15:17:30.394 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:30.395 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:30.395 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:30.401 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:30.415 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:30.420 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [Pipeline] // stage "is-owner": true, run task successfully test_expire_owner pass table test.t2 not exists for 9-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x [2024/05/06 15:17:30.608 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:30.619 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:30.624 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:30.626 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:30.714 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:30.861 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [Pipeline] // stage table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later [Pipeline] } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... [2024/05/06 15:17:31.297 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:31.304 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:31.310 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:31.404 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:31.409 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [2024/05/06 15:17:31.421 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:31.619 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:31.625 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:31.629 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:31.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check diff failed 10-th time, retry later [2024/05/06 15:17:31.702 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:31.811 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Mon May 6 15:17:31 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33603362.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76796236-3563-4850-9967-e6cc8ef30e76 {"id":"76796236-3563-4850-9967-e6cc8ef30e76","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc37a680c 76796236-3563-4850-9967-e6cc8ef30e76 /tidb/cdc/default/default/upstream/7365782293514767895 {"id":7365782293514767895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76796236-3563-4850-9967-e6cc8ef30e76 {"id":"76796236-3563-4850-9967-e6cc8ef30e76","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc37a680c 76796236-3563-4850-9967-e6cc8ef30e76 /tidb/cdc/default/default/upstream/7365782293514767895 {"id":7365782293514767895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/76796236-3563-4850-9967-e6cc8ef30e76 {"id":"76796236-3563-4850-9967-e6cc8ef30e76","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc37a680c 76796236-3563-4850-9967-e6cc8ef30e76 /tidb/cdc/default/default/upstream/7365782293514767895 {"id":7365782293514767895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 63-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:32.009 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 15:17:32.099 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] wait process cdc.test exit for 3-th time... table test.t2 not exists for 10-th check, retry later [2024/05/06 15:17:32.518 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:32.605 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 15:17:32.612 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:32.624 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4fbc4c8c_20d3_4b0c_bc74_237b4a72251a"] [2024/05/06 15:17:32.652 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:32.723 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbd3283bb_35b4_407b_adf4_de818d8157cd"] table partition_table.t2 not exists for 2-th check, retry later lease 22318f4cc37a680c revoked [2024/05/06 15:17:32.753 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 15:17:32.916 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs93956390_d5bd_4242_8826_8c09207c6315"] [2024/05/06 15:17:32.931 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:17:32 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> [2024/05/06 15:17:33.023 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs405bbee7_4d92_4fd5_ae18_ccb8f0d7e0e3"] [2024/05/06 15:17:33.107 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:33.107 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:33.195 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs32a218fc_97fa_484c_a7e2_688b84909639"] [2024/05/06 15:17:33.204 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:32.872 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2024/05/06 15:17:32.886 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark] [ts=449571678110875653] [2024/05/06 15:17:32.886 +08:00] [INFO] [case.go:498] ["all tables synced"] [endTs=449571678110875653] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:33.298 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4466bb67_3b0c_491e_9a37_b655b3100230"] [2024/05/06 15:17:33.596 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:33.596 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table capture_session_done_during_task.t exists [2024/05/06 15:17:33.918 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 64-th check, retry later check diff failed 1-th time, retry later check diff failed 11-th time, retry later [2024/05/06 15:17:34.025 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:34.103 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:34.105 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:34.125 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:34.126 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:34.133 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:34.133 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:34.216 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:34.219 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:34.305 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:34.313 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:34.503 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7bfa3194_6380_4e88_8a1a_b556a62204d7"] start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table partition_table.t2 not exists for 3-th check, retry later table test.t2 not exists for 11-th check, retry later [2024/05/06 15:17:35.004 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:35.015 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f5c0c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:3423, start at 2024-05-06 15:17:34.12692934 +0800 CST m=+7.435508971 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:34.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f5c0c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:3423, start at 2024-05-06 15:17:34.12692934 +0800 CST m=+7.435508971 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:34.144 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f5cbc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:3453, start at 2024-05-06 15:17:34.167212709 +0800 CST m=+7.365415991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:34.182 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:34.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:34.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e55f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571677724213251 MinTableBarrierTs:449571679034933252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571677724213251,"min-table-barrier-ts":449571679034933252,"admin-job-type":0} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e55f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571677724213251 MinTableBarrierTs:449571679034933252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571677724213251,"min-table-barrier-ts":449571679034933252,"admin-job-type":0} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.+ grep -q 'failed to get info:' 0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038e55f0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571677724213251 MinTableBarrierTs:449571679034933252 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571677724213251, checkpointTs: 449571677724213251, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571677724213251,"min-table-barrier-ts":449571679034933252,"admin-job-type":0} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2024/05/06 15:17:35.101 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:35.123 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:35.124 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:35.215 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:35.220 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:35.297 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:35.317 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:35.318 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:35.326 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:17:35.405 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:35.494 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:35.495 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] check diff failed 2-th time, retry later check diff failed 12-th time, retry later [2024/05/06 15:17:35.998 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:36.006 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:36.111 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:36.114 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs41772a56_4027_46fe_986d_b16a37450e87"] build mysql_test successfully run all mysql test cases time="2024-05-06T15:17:32+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2024-05-06T15:17:32+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000318000 {{} {} 0} {0 0} [0xc000332000] map[] 0 1 0xc00031c000 false map[0xc000332000:map[0xc000332000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 15:17:36.200 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:36.203 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:36.302 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:36.304 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:36.312 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table ddl_manager.finish_mark not exists for 65-th check, retry later "is-owner": true, run task successfully [2024/05/06 15:17:36.398 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:36.400 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:36.417 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:36.503 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:36.518 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:36.598 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table test.t2 exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [2024/05/06 15:17:36.906 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:36.912 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:37.198 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:37.205 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:37.208 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:37.301 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:37.305 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:37.500 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:37.506 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:37.514 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:37.604 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:37.614 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:37.695 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:37.706 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] wait process cdc.test exit for 2-th time... [2024/05/06 15:17:37.724 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:37.793 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:37.917 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 13-th time, retry later owner pid: 3365 owner id 5a9704f4-e97f-4ef3-9d93-087101334645 [Mon May 6 15:17:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.34713473.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:17:37.998 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:38.215 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 66-th check, retry later [2024/05/06 15:17:38.302 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:38.313 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:38.313 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:38.405 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table partition_table.finish_mark not exists for 2-th check, retry later check diff failed 3-th time, retry later [2024/05/06 15:17:38.707 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:38.712 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:38.728 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 4-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f9c480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:3720, start at 2024-05-06 15:17:38.201315303 +0800 CST m=+5.378334454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:38.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:38.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:38.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:38.822 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:38.902 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:38.913 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:38.927 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:38.949 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:38.996 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:39.113 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:39.113 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:39.199 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:39.215 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:39.216 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:39.222 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:39.224 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 5-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:17:39.406 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:39.409 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:39.610 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:39.705 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 6-th time... [2024/05/06 15:17:39.914 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:39.999 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:40.095 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:40.100 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 7-th time... process cdc.test already exit [Mon May 6 15:17:40 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> table ddl_manager.finish_mark not exists for 67-th check, retry later [2024/05/06 15:17:40.194 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:40.307 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:40.318 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:40.321 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff successfully check diff failed 1-th time, retry later check diff failed 14-th time, retry later [2024/05/06 15:17:40.406 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:40.413 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:40.496 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:40.506 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:40.516 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:40.519 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:40.707 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:40.713 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table partition_table.finish_mark not exists for 3-th check, retry later [2024/05/06 15:17:40.902 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:40.909 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:40.911 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:40.929 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:41.018 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:41.111 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:41.125 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036ee000 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571680345653255 MinTableBarrierTs:449571680608059400 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571680345653255,"min-table-barrier-ts":449571680608059400,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036ee000 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571680345653255 MinTableBarrierTs:449571680608059400 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571680345653255,"min-table-barrier-ts":449571680608059400,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 + grep -q 'failed to get info:' {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036ee000 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571680345653255 MinTableBarrierTs:449571680608059400 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571680345653255,"min-table-barrier-ts":449571680608059400,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "5a9704f4-e97f-4ef3-9d93-087101334645" | grep id Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:41.200 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:41.210 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:41.216 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:41.308 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:41.309 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:41.314 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:41.316 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f9c480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:3720, start at 2024-05-06 15:17:38.201315303 +0800 CST m=+5.378334454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:38.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:38.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:38.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d330f9c2c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:3802, start at 2024-05-06 15:17:38.198460847 +0800 CST m=+5.325515830 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:38.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:38.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:38.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:41.503 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:41.595 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:17:41 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.47014705.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:41.711 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:41.714 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:41.796 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:41.797 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:41.901 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:41.996 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:42.011 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:42.013 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:42.110 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:42.112 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] "cluster-id": "default" "id": "48ef49f3-1836-4ade-a18d-6e64116147e5", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 68-th check, retry later check diff failed 15-th time, retry later [2024/05/06 15:17:42.211 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:42.215 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:42.294 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:42.303 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:42.495 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:42.516 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:42.624 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:42.701 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:42.714 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:42.719 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:42.811 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:42.904 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:42.909 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table partition_table.finish_mark not exists for 4-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... [2024/05/06 15:17:42.917 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:43.094 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:43.097 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:43.098 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:43.101 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:17:43 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.51715173.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:43.210 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:43.211 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:43.317 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:43.419 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process cdc.test exit for 2-th time... capture_id: 48ef49f3-1836-4ade-a18d-6e64116147e5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:43.598 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:43.611 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:43.611 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:43.616 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:43.716 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:43.804 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:43.807 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:43.813 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:43.913 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:43 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [2024/05/06 15:17:43.995 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:43.996 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:44.001 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:44.114 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:44.118 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] check diff failed 16-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:44 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/772590db-2bec-4e4e-823b-49b4116dde74 {"id":"772590db-2bec-4e4e-823b-49b4116dde74","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3a9dafb 772590db-2bec-4e4e-823b-49b4116dde74 /tidb/cdc/default/default/upstream/7365782342528384771 {"id":7365782342528384771,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/772590db-2bec-4e4e-823b-49b4116dde74 {"id":"772590db-2bec-4e4e-823b-49b4116dde74","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3a9dafb 772590db-2bec-4e4e-823b-49b4116dde74 /tidb/cdc/default/default/upstream/7365782342528384771 {"id":7365782342528384771,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/772590db-2bec-4e4e-823b-49b4116dde74 {"id":"772590db-2bec-4e4e-823b-49b4116dde74","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3a9dafb 772590db-2bec-4e4e-823b-49b4116dde74 /tidb/cdc/default/default/upstream/7365782342528384771 {"id":7365782342528384771,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4741.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test [2024/05/06 15:17:44.233 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:44.341 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:44.514 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:44.514 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:44.594 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:44.600 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table ddl_manager.finish_mark not exists for 69-th check, retry later table partition_table.finish_mark not exists for 5-th check, retry later [2024/05/06 15:17:44.702 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:44.717 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:44.719 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:44.797 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:44.814 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:44.819 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:44.822 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:44.913 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:44.919 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] Create changefeed successfully! ID: test Info: {"upstream_id":7365782342528384771,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T15:17:44.82421179+08:00","start_ts":449571681633566726,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571681633566726,"checkpoint_ts":449571681633566726,"checkpoint_time":"2024-05-06 15:17:44.630"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:45.205 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:45.213 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:45.216 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:45.311 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:45.323 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:45.399 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:45.403 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:45.422 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:45.427 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:45.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:45.508 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:45.516 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:45.805 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:45.813 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:45.813 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:45.907 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:45.998 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:46.006 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:46.016 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:46.029 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:46.029 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:46.034 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:46.111 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:46.118 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:46.396 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:46.402 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:46.406 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:46.502 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:46 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4fc5b20a-045c-4376-b92d-891e4589e537 {"id":"4fc5b20a-045c-4376-b92d-891e4589e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3be0b07 4fc5b20a-045c-4376-b92d-891e4589e537 /tidb/cdc/default/default/upstream/7365782361647113701 {"id":7365782361647113701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4fc5b20a-045c-4376-b92d-891e4589e537 {"id":"4fc5b20a-045c-4376-b92d-891e4589e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3be0b07 4fc5b20a-045c-4376-b92d-891e4589e537 /tidb/cdc/default/default/upstream/7365782361647113701 {"id":7365782361647113701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4fc5b20a-045c-4376-b92d-891e4589e537 {"id":"4fc5b20a-045c-4376-b92d-891e4589e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3be0b07 4fc5b20a-045c-4376-b92d-891e4589e537 /tidb/cdc/default/default/upstream/7365782361647113701 {"id":7365782361647113701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.5217.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table ddl_manager.finish_mark not exists for 70-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later [2024/05/06 15:17:46.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:46.701 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:46.725 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 15:17:46.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] check diff failed 17-th time, retry later 0 [Mon May 6 15:17:46 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35953597.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: a8afb82f-4f56-421e-bbf3-d4b184ed746f Info: {"upstream_id":7365782361647113701,"namespace":"default","id":"a8afb82f-4f56-421e-bbf3-d4b184ed746f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:46.676081209+08:00","start_ts":449571682135310338,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571682135310338,"checkpoint_ts":449571682135310338,"checkpoint_time":"2024-05-06 15:17:46.544"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [2024/05/06 15:17:47.005 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:47.018 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:47.027 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 15:17:47.110 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:47.211 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 15:17:47.310 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 15:17:47.324 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 15:17:47.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 15:17:47.328 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf7374e1f_1532_43f2_800f_19d492030853"] [2024/05/06 15:17:47.507 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7232a93a_4d0c_4bdc_9843_adbf2589bbb2"] [2024/05/06 15:17:47.632 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 15:17:47.730 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfac6af97_503b_476a_a602_b1a9489a198b"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331020bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:3900, start at 2024-05-06 15:17:46.710490924 +0800 CST m=+5.248176896 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:46.717 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:46.721 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:46.721 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:47.835 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 15:17:47.916 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 15:17:47.997 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:48.010 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:48.036 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4780.out cli changefeed pause -c test [2024/05/06 15:17:48.052 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:46.722 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark1000] [2024/05/06 15:17:46.722 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark1000] split_and_random_merge scale: 20 table ddl_manager.finish_mark not exists for 71-th check, retry later [2024/05/06 15:17:48.296 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:48.307 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:48.443 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:48.504 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:48.513 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2fe0f87f_4df2_4f5f_958d_b64edd5c7061"] [2024/05/06 15:17:48.529 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table partition_table.finish_mark not exists for 7-th check, retry later check diff failed 18-th time, retry later [2024/05/06 15:17:48.607 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:48.722 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:17:48.833 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:48.834 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:49.016 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:49.023 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:49.032 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 15:17:49.043 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check_redo_resolved_ts 73723af2-c490-450e-a190-817e0e483b07 449571682618441733 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/73723af2-c490-450e-a190-817e0e483b07/meta global resolved ts 449571677060988934 not forward to 449571682618441733 run task failed 1-th time, retry later [2024/05/06 15:17:49.093 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:49.139 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:49.295 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs912230ec_afed_46f7_bb6e_f3539e9fe083"] [2024/05/06 15:17:49.445 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:49.504 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331037c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:4045, start at 2024-05-06 15:17:48.165815989 +0800 CST m=+5.179771104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:48.172 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:48.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:48.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331037c00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:4045, start at 2024-05-06 15:17:48.165815989 +0800 CST m=+5.179771104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:48.172 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:48.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:48.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331037c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:4114, start at 2024-05-06 15:17:48.175837337 +0800 CST m=+5.130172441 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:48.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:48.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:48.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:49.709 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:49.713 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:49.722 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:49.806 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:49.818 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:49.820 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark2000] [2024/05/06 15:17:49.820 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark2000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331020bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:3900, start at 2024-05-06 15:17:46.710490924 +0800 CST m=+5.248176896 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:46.717 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:46.721 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:46.721 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331022240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:3984, start at 2024-05-06 15:17:46.795429132 +0800 CST m=+5.272535487 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:46.802 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:46.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:46.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:17:49.830 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:49.933 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 15:17:49.933 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034985a0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571681132085258 MinTableBarrierTs:449571681132085258 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5af6be65-c887-4d82-955a-e6e5b8a97894 {"id":"5af6be65-c887-4d82-955a-e6e5b8a97894","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f815b1 5af6be65-c887-4d82-955a-e6e5b8a97894 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571681132085258,"min-table-barrier-ts":449571681132085258,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5af6be65-c887-4d82-955a-e6e5b8a97894/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034985a0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571681132085258 MinTableBarrierTs:449571681132085258 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5af6be65-c887-4d82-955a-e6e5b8a97894 {"id":"5af6be65-c887-4d82-955a-e6e5b8a97894","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f815b1 5af6be65-c887-4d82-955a-e6e5b8a97894 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571681132085258,"min-table-barrier-ts":449571681132085258,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5af6be65-c887-4d82-955a-e6e5b8a97894/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0034985a0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571681132085258 MinTableBarrierTs:449571681132085258 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48ef49f3-1836-4ade-a18d-6e64116147e5 {"id":"48ef49f3-1836-4ade-a18d-6e64116147e5","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5a9704f4-e97f-4ef3-9d93-087101334645 {"id":"5a9704f4-e97f-4ef3-9d93-087101334645","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/5af6be65-c887-4d82-955a-e6e5b8a97894 {"id":"5af6be65-c887-4d82-955a-e6e5b8a97894","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81546 5a9704f4-e97f-4ef3-9d93-087101334645 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81574 48ef49f3-1836-4ade-a18d-6e64116147e5 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f815b1 5af6be65-c887-4d82-955a-e6e5b8a97894 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571681132085258,"min-table-barrier-ts":449571681132085258,"admin-job-type":0} /tidb/cdc/default/default/task/position/48ef49f3-1836-4ade-a18d-6e64116147e5/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5a9704f4-e97f-4ef3-9d93-087101334645/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5af6be65-c887-4d82-955a-e6e5b8a97894/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:50.109 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:50.195 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:50.326 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table ddl_manager.finish_mark exists [2024/05/06 15:17:50.401 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:50.410 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:50.507 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff failed 19-th time, retry later check_redo_resolved_ts 73723af2-c490-450e-a190-817e0e483b07 449571682618441733 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/73723af2-c490-450e-a190-817e0e483b07/meta check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4795.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm [2024/05/06 15:17:50.738 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2024/05/06 15:17:50.597 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:50.605 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:50.717 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 15:17:50.723 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 15:17:50.813 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} [2024/05/06 15:17:50.921 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark1000] [ts=449571682881896453] [2024/05/06 15:17:50.921 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark1000] [2024/05/06 15:17:50.921 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571682881896453] [2024/05/06 15:17:50.925 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:17:50.930 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:17:50.936 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:17:50.941 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table partition_table.finish_mark not exists for 8-th check, retry later [2024/05/06 15:17:50.900 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] wait process cdc.test exit for 1-th time... Update changefeed config successfully! ID: test Info: {"upstream_id":7365782342528384771,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-06T15:17:44.82421179+08:00","start_ts":449571681633566726,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571682170961948,"checkpoint_ts":449571682170961948,"checkpoint_time":"2024-05-06 15:17:46.680"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:51.094 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:51.111 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:51.114 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:51.209 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:51.229 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:51.239 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33105c000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:9990, start at 2024-05-06 15:17:50.487807683 +0800 CST m=+9.656210593 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:50.495 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:50.464 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:50.464 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33105c000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:9990, start at 2024-05-06 15:17:50.487807683 +0800 CST m=+9.656210593 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:50.495 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:50.464 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:50.464 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331043c80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:10065, start at 2024-05-06 15:17:48.948982786 +0800 CST m=+8.058723542 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:19:48.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:17:49.065 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:07:49.065 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 15:17:51 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.53795381.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:17:51.397 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 15:17:51.399 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 15:17:51.502 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:51.515 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 2-th time... [2024/05/06 15:17:51.612 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:51.626 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:51.707 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:51.731 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:51.809 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:51.811 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:51.820 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 3-th time... [2024/05/06 15:17:51.910 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 15:17:51.918 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 15:17:52.016 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:52.030 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:52.112 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] run task failed 2-th time, retry later [2024/05/06 15:17:52.129 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:52.217 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:52.310 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:52.319 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:52.330 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Starting Upstream TiDB... wait process cdc.test exit for 4-th time... [2024/05/06 15:17:52.427 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 15:17:52.513 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 15:17:52.597 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:52.623 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:17:52.710 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 15:17:52.832 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:52.897 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:52.916 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table partition_table.finish_mark not exists for 9-th check, retry later check diff failed 20-th time, retry later [2024/05/06 15:17:52.936 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 15:17:53.038 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 15:17:53.041 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 15:17:53.141 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4810.out cli changefeed resume -c test wait process cdc.test exit for 5-th time... [Mon May 6 15:17:52 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.53805382.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11369.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 15:17:53.200 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 15:17:53.494 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:53.520 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 15:17:53.547 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:53.608 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 6-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:17:53.835 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 15:17:53.731 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 15:17:53.735 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 15:17:53.819 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:53.827 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:17:53.958 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark3000] [2024/05/06 15:17:53.958 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark3000] ./t/alter_table.test: ok! 70 test cases passed, take time 20.430347363 s time="2024-05-06T15:17:52+08:00" level=info msg="run test [alter_table] ok" time="2024-05-06T15:17:52+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000388298 {{} {} 0} {0 0} [0xc000136480] map[] 0 1 0xc000380720 false map[0xc000136480:map[0xc000136480:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 15:17:54.032 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:54.054 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 15:17:54.096 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process cdc.test exit for 7-th time... [2024/05/06 15:17:54.293 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:54.319 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 15:17:54.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 8-th time... Verifying downstream PD is started... [2024/05/06 15:17:54.508 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 15:17:54.523 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 15:17:54.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:54.696 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28c35a4a-f391-4a18-9f5e-bd5382f0e83f {"id":"28c35a4a-f391-4a18-9f5e-bd5382f0e83f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e5dff1 28c35a4a-f391-4a18-9f5e-bd5382f0e83f /tidb/cdc/default/default/upstream/7365782409225264661 {"id":7365782409225264661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28c35a4a-f391-4a18-9f5e-bd5382f0e83f {"id":"28c35a4a-f391-4a18-9f5e-bd5382f0e83f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e5dff1 28c35a4a-f391-4a18-9f5e-bd5382f0e83f /tidb/cdc/default/default/upstream/7365782409225264661 {"id":7365782409225264661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28c35a4a-f391-4a18-9f5e-bd5382f0e83f {"id":"28c35a4a-f391-4a18-9f5e-bd5382f0e83f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e5dff1 28c35a4a-f391-4a18-9f5e-bd5382f0e83f /tidb/cdc/default/default/upstream/7365782409225264661 {"id":7365782409225264661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x run task successfully table partition_table.finish_mark not exists for 10-th check, retry later [2024/05/06 15:17:54.720 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 15:17:54.735 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 15:17:54.853 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] wait process cdc.test exit for 1-th time... check diff successfully wait process cdc.test exit for 9-th time... [2024/05/06 15:17:55.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x wait process cdc.test exit for 10-th time... [2024/05/06 15:17:55.654 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v1m23.259909373s"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + set +x + tso='449571683949346818 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571683949346818 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:17:55 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1142511427.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2024/05/06 15:17:55.850 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2024/05/06 15:17:55.958 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a931c5f-5269-49d5-ad73-89154bab806e {"id":"6a931c5f-5269-49d5-ad73-89154bab806e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3dc072c 6a931c5f-5269-49d5-ad73-89154bab806e /tidb/cdc/default/default/upstream/7365782406182130975 {"id":7365782406182130975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a931c5f-5269-49d5-ad73-89154bab806e {"id":"6a931c5f-5269-49d5-ad73-89154bab806e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3dc072c 6a931c5f-5269-49d5-ad73-89154bab806e /tidb/cdc/default/default/upstream/7365782406182130975 {"id":7365782406182130975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a931c5f-5269-49d5-ad73-89154bab806e {"id":"6a931c5f-5269-49d5-ad73-89154bab806e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3dc072c 6a931c5f-5269-49d5-ad73-89154bab806e /tidb/cdc/default/default/upstream/7365782406182130975 {"id":7365782406182130975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5428.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: 22472085-0937-498d-9d42-d16fb7d16740 Info: {"upstream_id":7365782406182130975,"namespace":"default","id":"22472085-0937-498d-9d42-d16fb7d16740","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:56.274266796+08:00","start_ts":449571684645863428,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571684645863428,"checkpoint_ts":449571684645863428,"checkpoint_time":"2024-05-06 15:17:56.121"} PASS cdc.test: no process found wait process cdc.test exit for 11-th time... process cdc.test already exit [Mon May 6 15:17:56 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> [2024/05/06 15:17:56.166 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 1m23.772693886s"] cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:17:56 CST 2024] <<<<<< run test case clustered_index success! >>>>>> *************************** 1. row *************************** count(distinct region_id): 1 start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 15:17:56.547 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:17:56.548 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:17:56.548 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571677100572685] [resolvedTs=449571683274063887] [2024/05/06 15:17:56.550 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:17:56.550 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:17:56.554 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=6.206062ms] [2024/05/06 15:17:56.558 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:17:56.562 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:17:56.563 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:17:56.562 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:17:56.564 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:17:56.564 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:17:56.570 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:17:56.572 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] table partition_table.finish_mark not exists for 11-th check, retry later [2024/05/06 15:17:56.824 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark4000] [2024/05/06 15:17:56.825 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark4000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4836.out cli changefeed pause -c test Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:17:57.182 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=634.47805ms] [2024/05/06 15:17:57.183 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449571677100572679,\"CommitTs\":449571677100572685,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_file\",\"tbl-name\":\"t1\",\"tbl-id\":108,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:17:57.864 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] + set +x table batch_add_table.finish_mark not exists for 1-th check, retry later [2024/05/06 15:17:57.940 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark2000] [ts=449571684671029253] [2024/05/06 15:17:57.940 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark2000] [2024/05/06 15:17:57.940 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571684671029253] [2024/05/06 15:17:57.943 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:17:58.020 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2024/05/06 15:17:58.021 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:17:58.027 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:17:58.033 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:17:58.152 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571683274063887] [2024/05/06 15:17:58.152 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571683274063887] [2024/05/06 15:17:58.252 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571683274063887] [2024/05/06 15:17:58.252 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571683274063887] table partition_table.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later [2024/05/06 15:17:58.824 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:17:58 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a2a6856-2b81-41a6-8707-7fdbac31ef4c {"id":"9a2a6856-2b81-41a6-8707-7fdbac31ef4c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e12755 9a2a6856-2b81-41a6-8707-7fdbac31ef4c /tidb/cdc/default/default/upstream/7365782400118383158 {"id":7365782400118383158,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a2a6856-2b81-41a6-8707-7fdbac31ef4c {"id":"9a2a6856-2b81-41a6-8707-7fdbac31ef4c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e12755 9a2a6856-2b81-41a6-8707-7fdbac31ef4c /tidb/cdc/default/default/upstream/7365782400118383158 {"id":7365782400118383158,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a2a6856-2b81-41a6-8707-7fdbac31ef4c {"id":"9a2a6856-2b81-41a6-8707-7fdbac31ef4c","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc3e12755 9a2a6856-2b81-41a6-8707-7fdbac31ef4c /tidb/cdc/default/default/upstream/7365782400118383158 {"id":7365782400118383158,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11471.out cli changefeed create --start-ts=449571683949346818 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365782400118383158,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:59.314075756+08:00","start_ts":449571683949346818,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571683949346818,"checkpoint_ts":449571683949346818,"checkpoint_time":"2024-05-06 15:17:53.464"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4850.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} [2024/05/06 15:17:59.414 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449571683274063887] [2024/05/06 15:17:59.414 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449571683274063887] [2024/05/06 15:17:59.514 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449571683274063887] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:17:59.515 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] Apply redo log successfully Update changefeed config successfully! ID: test Info: {"upstream_id":7365782342528384771,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T15:17:44.82421179+08:00","start_ts":449571681633566726,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571684884152322,"checkpoint_ts":449571682249605124,"checkpoint_time":"2024-05-06 15:17:46.980"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff successfully [Mon May 6 15:18:00 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 table batch_add_table.finish_mark not exists for 2-th check, retry later [2024/05/06 15:18:00.039 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] split_and_random_merge scale: 40 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3310f6d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:7393, start at 2024-05-06 15:18:00.409061533 +0800 CST m=+6.017850354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:00.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:00.422 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:00.422 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:00.837 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later table partition_table.finish_mark exists check diff successfully [2024/05/06 15:18:01.127 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark5000] [2024/05/06 15:18:01.127 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark5000] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.4865.out cli changefeed resume -c test [2024/05/06 15:18:02.051 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table batch_add_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:02 CST 2024] <<<<<< run test case partition_table success! >>>>>> PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3310f6d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:7393, start at 2024-05-06 15:18:00.409061533 +0800 CST m=+6.017850354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:00.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:00.422 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:00.422 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3310fa900008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:7468, start at 2024-05-06 15:18:00.622458928 +0800 CST m=+6.170547853 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:00.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:00.612 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:00.612 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table mark.finish_mark_2 not exists for 3-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.`simple-dash` not exists for 2-th check, retry later [2024/05/06 15:18:02.849 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/06 15:18:03.127 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:03 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> [2024/05/06 15:18:04.063 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass [2024/05/06 15:18:04.221 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark3000] [ts=449571686446792709] [2024/05/06 15:18:04.221 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark3000] [2024/05/06 15:18:04.221 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571686446792709] [2024/05/06 15:18:04.224 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:04.231 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:04.241 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:04.322 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark6000] [2024/05/06 15:18:04.322 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark6000] wait process cdc.test exit for 1-th time... + set +x table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:04.326 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table mark.finish_mark_2 not exists for 4-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8773.out cli tso query --pd=http://127.0.0.1:2379 table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s wait process cdc.test exit for 2-th time... + info='{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571686832930822, "checkpoint_time": "2024-05-06 15:18:04.464", "error": null }' + echo '{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571686832930822, "checkpoint_time": "2024-05-06 15:18:04.464", "error": null }' { "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571686832930822, "checkpoint_time": "2024-05-06 15:18:04.464", "error": null } ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571686832930822, '"checkpoint_time":' '"2024-05-06' '15:18:04.464",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571686832930822, '"checkpoint_time":' '"2024-05-06' '15:18:04.464",' '"error":' null '}' ++ jq -r .error.message [2024/05/06 15:18:04.917 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/06 15:18:05.140 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_retryable_error [Mon May 6 15:18:05 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.38263828.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:18:06.322 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] table mark.finish_mark_2 not exists for 5-th check, retry later + set +x + tso='449571686979207172 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571686979207172 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:18:06 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.88098811.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11732.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:18:06.934 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] *************************** 1. row *************************** count(distinct region_id): 1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331136800018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:23253, start at 2024-05-06 15:18:04.477685089 +0800 CST m=+5.637890064 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:04.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:04.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:04.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331136800018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:23253, start at 2024-05-06 15:18:04.477685089 +0800 CST m=+5.637890064 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:04.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:04.448 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:04.448 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33113744000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:23329, start at 2024-05-06 15:18:04.516114985 +0800 CST m=+5.594418577 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:04.531 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:04.497 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:04.497 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:18:07.219 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2024/05/06 15:17:55.499 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 20"] [2024/05/06 15:18:00.502 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 29"] table batch_update_to_no_batch.recover_and_insert not exists for 3-th check, retry later [2024/05/06 15:18:07.933 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark7000] [2024/05/06 15:18:07.933 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark7000] [2024/05/06 15:18:08.334 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004036120 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571686388072449 MinTableBarrierTs:449571687698792452 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571686388072449,"min-table-barrier-ts":449571687698792452,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004036120 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571686388072449 MinTableBarrierTs:449571687698792452 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571686388072449,"min-table-barrier-ts":449571687698792452,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004036120 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571686388072449 MinTableBarrierTs:449571687698792452 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571686388072449, checkpointTs: 449571686388072449, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571686388072449,"min-table-barrier-ts":449571687698792452,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331163f8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:23467, start at 2024-05-06 15:18:07.370179834 +0800 CST m=+5.124421889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:07.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:07.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:07.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331163f8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:23467, start at 2024-05-06 15:18:07.370179834 +0800 CST m=+5.124421889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:07.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:07.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:07.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331164ac0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-7d19v, pid:23550, start at 2024-05-06 15:18:07.407163928 +0800 CST m=+5.111937178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:07.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:07.403 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:07.403 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ./t/bigint.test: ok! 138 test cases passed, take time 12.86139716 s time="2024-05-06T15:18:06+08:00" level=info msg="run test [bigint] ok" time="2024-05-06T15:18:06+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003881a8 {{} {} 0} {0 0} [0xc0004d8e10] map[] 0 1 0xc0003806c0 false map[0xc0004d8e10:map[0xc0004d8e10:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" ./t/composite_index.test: ok! 12 test cases passed, take time 1.545521278 s time="2024-05-06T15:18:08+08:00" level=info msg="run test [composite_index] ok" time="2024-05-06T15:18:08+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000ca0d8 {{} {} 0} {0 0} [0xc0004d8ea0] map[] 0 1 0xc00003e3c0 false map[0xc0004d8ea0:map[0xc0004d8ea0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 15:18:08.946 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] table mark.finish_mark_2 not exists for 6-th check, retry later [2024/05/06 15:18:09.232 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24701.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03be9c5d-08c5-4b5a-b636-f1c7da801b19 {"id":"03be9c5d-08c5-4b5a-b636-f1c7da801b19","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4075ff9 03be9c5d-08c5-4b5a-b636-f1c7da801b19 /tidb/cdc/default/default/upstream/7365782452811885751 {"id":7365782452811885751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03be9c5d-08c5-4b5a-b636-f1c7da801b19 {"id":"03be9c5d-08c5-4b5a-b636-f1c7da801b19","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4075ff9 03be9c5d-08c5-4b5a-b636-f1c7da801b19 /tidb/cdc/default/default/upstream/7365782452811885751 {"id":7365782452811885751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03be9c5d-08c5-4b5a-b636-f1c7da801b19 {"id":"03be9c5d-08c5-4b5a-b636-f1c7da801b19","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4075ff9 03be9c5d-08c5-4b5a-b636-f1c7da801b19 /tidb/cdc/default/default/upstream/7365782452811885751 {"id":7365782452811885751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8863.out cli changefeed create --start-ts=449571686979207172 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ "is-owner": true, run task successfully table batch_update_to_no_batch.recover_and_insert not exists for 4-th check, retry later Create changefeed successfully! ID: 9f790cc2-3456-4821-a978-0864c65f18af Info: {"upstream_id":7365782452811885751,"namespace":"default","id":"9f790cc2-3456-4821-a978-0864c65f18af","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:18:09.905565948+08:00","start_ts":449571686979207172,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571686979207172,"checkpoint_ts":449571686979207172,"checkpoint_time":"2024-05-06 15:18:05.022"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:18:09.934 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.023866808 s time="2024-05-06T15:18:09+08:00" level=info msg="run test [concurrent_ddl] ok" time="2024-05-06T15:18:09+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0004860c8 {{} {} 0} {0 0} [0xc000336090] map[] 0 1 0xc000480120 false map[0xc000336090:map[0xc000336090:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" [2024/05/06 15:18:10.346 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24890.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... table mark.finish_mark_2 not exists for 7-th check, retry later [2024/05/06 15:18:11.017 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2024/05/06 15:18:11.036 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark4000] [ts=449571688157806597] [2024/05/06 15:18:11.036 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark4000] [2024/05/06 15:18:11.036 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571688157806597] [2024/05/06 15:18:11.041 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] + set +x + tso='449571688152563713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571688152563713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 15:18:11.118 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark8000] [2024/05/06 15:18:11.118 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark8000] [2024/05/06 15:18:11.120 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:11.125 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:11.130 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:11.317 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] split_and_random_merge scale: 80 owner pid: 3831 owner id 8e4cc8e4-e555-4e6e-8002-1dd774198be7 [Mon May 6 15:18:11 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.39353937.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ***************** properties ***************** "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="move_table" "readallfields"="true" "insertproportion"="0" "scanproportion"="0" "readproportion"="0" "dotransactions"="false" "recordcount"="10000" "updateproportion"="0" "threadcount"="10" ********************************************** + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449571687357218823, "checkpoint_time": "2024-05-06 15:18:06.464", "error": null }' + echo '{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449571687357218823, "checkpoint_time": "2024-05-06 15:18:06.464", "error": null }' { "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449571687357218823, "checkpoint_time": "2024-05-06 15:18:06.464", "error": null } ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449571687357218823, '"checkpoint_time":' '"2024-05-06' '15:18:06.464",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449571687357218823, '"checkpoint_time":' '"2024-05-06' '15:18:06.464",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11796.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [2024/05/06 15:18:12.028 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] \033[0;36m<<< Run all test success >>>\033[0m Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:18:12.523 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [Pipeline] // cache table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later [Pipeline] } + set +x + tso='449571688509341697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571688509341697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:18:12 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2492124923.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365782400118383158,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:17:59.314075756+08:00","start_ts":449571683949346818,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571687357218823,"checkpoint_ts":449571687357218823,"checkpoint_time":"2024-05-06 15:18:06.464"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... [Pipeline] // dir table mark.finish_mark_2 not exists for 8-th check, retry later [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // timeout table ddl_sequence.finish_mark not exists for 1-th check, retry later [Pipeline] } [2024/05/06 15:18:13.118 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 15:18:13.334 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [2024/05/06 15:18:14.040 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later [2024/05/06 15:18:14.537 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] Run finished, takes 2.509066714s INSERT - Takes(s): 2.5, Count: 10000, OPS: 3995.5, Avg(us): 2459, Min(us): 767, Max(us): 61676, 95th(us): 3000, 99th(us): 41000 [Mon May 6 15:18:14 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2478724789.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036761b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689271918601 MinTableBarrierTs:449571689271918601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f91f6b53-bc42-4ecc-923a-07a0c51adc1b {"id":"f91f6b53-bc42-4ecc-923a-07a0c51adc1b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81643 f91f6b53-bc42-4ecc-923a-07a0c51adc1b /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689271918601,"min-table-barrier-ts":449571689271918601,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f91f6b53-bc42-4ecc-923a-07a0c51adc1b/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036761b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689271918601 MinTableBarrierTs:449571689271918601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f91f6b53-bc42-4ecc-923a-07a0c51adc1b {"id":"f91f6b53-bc42-4ecc-923a-07a0c51adc1b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81643 f91f6b53-bc42-4ecc-923a-07a0c51adc1b /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689271918601,"min-table-barrier-ts":449571689271918601,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f91f6b53-bc42-4ecc-923a-07a0c51adc1b/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert+ grep -q 'failed to get info:' -allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036761b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689271918601 MinTableBarrierTs:449571689271918601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e4cc8e4-e555-4e6e-8002-1dd774198be7 {"id":"8e4cc8e4-e555-4e6e-8002-1dd774198be7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f91f6b53-bc42-4ecc-923a-07a0c51adc1b {"id":"f91f6b53-bc42-4ecc-923a-07a0c51adc1b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81618 8e4cc8e4-e555-4e6e-8002-1dd774198be7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81643 f91f6b53-bc42-4ecc-923a-07a0c51adc1b /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689271918601,"min-table-barrier-ts":449571689271918601,"admin-job-type":0} /tidb/cdc/default/default/task/position/8e4cc8e4-e555-4e6e-8002-1dd774198be7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f91f6b53-bc42-4ecc-923a-07a0c51adc1b/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "8e4cc8e4-e555-4e6e-8002-1dd774198be7" | grep id [2024/05/06 15:18:14.839 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark9000] [2024/05/06 15:18:14.840 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark9000] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11835.out cli changefeed --changefeed-id custom-changefeed-name resume Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:15.134 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 15:18:15.348 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... "cluster-id": "default" "id": "f91f6b53-bc42-4ecc-923a-07a0c51adc1b", "cluster-id": "default" run task successfully Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_sequence.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:15 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaea0191-8e04-4752-89ae-8549ad5aa222 {"id":"aaea0191-8e04-4752-89ae-8549ad5aa222","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc42d37f5 aaea0191-8e04-4752-89ae-8549ad5aa222 /tidb/cdc/default/default/upstream/7365782496534842227 {"id":7365782496534842227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaea0191-8e04-4752-89ae-8549ad5aa222 {"id":"aaea0191-8e04-4752-89ae-8549ad5aa222","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc42d37f5 aaea0191-8e04-4752-89ae-8549ad5aa222 /tidb/cdc/default/default/upstream/7365782496534842227 {"id":7365782496534842227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaea0191-8e04-4752-89ae-8549ad5aa222 {"id":"aaea0191-8e04-4752-89ae-8549ad5aa222","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc42d37f5 aaea0191-8e04-4752-89ae-8549ad5aa222 /tidb/cdc/default/default/upstream/7365782496534842227 {"id":7365782496534842227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24966.out cli changefeed create --start-ts=449571688509341697 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: 56bd443a-94bf-4c87-bad8-2db4da6bf91b Info: {"upstream_id":7365782496534842227,"namespace":"default","id":"56bd443a-94bf-4c87-bad8-2db4da6bf91b","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T15:18:15.75742713+08:00","start_ts":449571688509341697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571688509341697,"checkpoint_ts":449571688509341697,"checkpoint_time":"2024-05-06 15:18:10.859"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... [2024/05/06 15:18:16.052 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:18:16.617 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/06 15:18:16.917 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] + set +x capture_id: f91f6b53-bc42-4ecc-923a-07a0c51adc1b % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 24840 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '8e4cc8e4-e555-4e6e-8002-1dd774198be7' 8e4cc8e4-e555-4e6e-8002-1dd774198be7 run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 2-th check, retry later + set +x [2024/05/06 15:18:17.218 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 15:18:17.422 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] wait process cdc.test exit for 1-th time... [2024/05/06 15:18:17.520 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark5000] [ts=449571689933570053] [2024/05/06 15:18:17.520 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark5000] [2024/05/06 15:18:17.520 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571689933570053] [2024/05/06 15:18:17.523 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:17.527 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:17.532 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:17.537 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table ddl_sequence.finish_mark not exists for 3-th check, retry later *************************** 1. row *************************** count(distinct region_id): 4 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:17 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24835.out cli changefeed create --start-ts=449571688152563713 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' Create changefeed successfully! ID: 7dbb39be-a55a-476d-b563-2ea64653043f Info: {"upstream_id":7365782470846785413,"namespace":"default","id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:18:18.031826409+08:00","start_ts":449571688152563713,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571688152563713,"checkpoint_ts":449571688152563713,"checkpoint_time":"2024-05-06 15:18:09.498"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... [2024/05/06 15:18:18.117 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later ./t/date_formats.test: ok! 99 test cases passed, take time 6.661789755 s time="2024-05-06T15:18:17+08:00" level=info msg="run test [date_formats] ok" time="2024-05-06T15:18:17+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000306048 {{} {} 0} {0 0} [0xc00052a3f0] map[] 0 1 0xc000316420 false map[0xc00052a3f0:map[0xc00052a3f0:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Mon May 6 15:18:18 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.40774079.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:18:18.720 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/06 15:18:18.724 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark10000] [2024/05/06 15:18:18.724 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark10000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used table mark.finish_mark_3 not exists for 3-th check, retry later [2024/05/06 15:18:18.933 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/06 15:18:19.319 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] + set +x [Mon May 6 15:18:19 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2487724879.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_sequence.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571690778198017, "checkpoint_time": "2024-05-06 15:18:19.514", "error": null }' + echo '{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571690778198017, "checkpoint_time": "2024-05-06 15:18:19.514", "error": null }' { "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571690778198017, "checkpoint_time": "2024-05-06 15:18:19.514", "error": null } ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571690778198017, '"checkpoint_time":' '"2024-05-06' '15:18:19.514",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571690778198017, '"checkpoint_time":' '"2024-05-06' '15:18:19.514",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11901.out cli changefeed --changefeed-id custom-changefeed-name remove [2024/05/06 15:18:20.132 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_update_to_no_batch.finish_mark not exists for 5-th check, retry later [2024/05/06 15:18:20.725 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:20.732 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 4-th check, retry later [2024/05/06 15:18:20.945 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449571690778198017 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:18:21.332 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] table ddl_sequence.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33122b9c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:4452, start at 2024-05-06 15:18:20.143949371 +0800 CST m=+5.142221852 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:20.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:20.135 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:20.135 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33122b9c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:4452, start at 2024-05-06 15:18:20.143949371 +0800 CST m=+5.142221852 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:20.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:20.135 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:20.135 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33122c340002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-jlvhl, pid:4533, start at 2024-05-06 15:18:20.17549976 +0800 CST m=+5.097486883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:20.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:20.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:20.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table big_txn.usertable1 exists Debugger for raftstore-v2 is used [2024/05/06 15:18:21.644 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark11000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:22.144 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] + set +x [2024/05/06 15:18:22.736 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:22.745 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2024/05/06 15:18:22.758 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark6000] [ts=449571689933570053] [2024/05/06 15:18:22.759 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark6000] [2024/05/06 15:18:22.759 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571689933570053] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbe510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688807923850 MinTableBarrierTs:449571691403149328 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688807923850,"min-table-barrier-ts":449571691403149328,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbe510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688807923850 MinTableBarrierTs:449571691403149328 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688807923850,"min-table-barrier-ts":449571691403149328,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbe510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688807923850 MinTableBarrierTs:449571691403149328 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688807923850,"min-table-barrier-ts":449571691403149328,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:18:22 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2494524947.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:22.829 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:22.832 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:22.846 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:22.852 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:22.852 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark11000] table region_merge.t1 exists check diff failed 1-th time, retry later [2024/05/06 15:18:23.031 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] table batch_update_to_no_batch.finish_mark exists check diff successfully [Mon May 6 15:18:23 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.57745776.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:18:23.418 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] wait process cdc.test exit for 1-th time... table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/06 15:18:24.218 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] table mark.finish_mark_3 not exists for 6-th check, retry later wait process cdc.test exit for 3-th time... table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:24.818 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:24.852 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:25.045 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00329afc0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689795944453 MinTableBarrierTs:449571690058088458 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689795944453,"min-table-barrier-ts":449571690058088458,"admin-job-type":0} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00329afc0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689795944453 MinTableBarrierTs:449571690058088458 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689795944453,"min-table-barrier-ts":449571690058088458,"admin-job-type":0} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00329afc0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571689795944453 MinTableBarrierTs:449571690058088458 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571689795944453,"min-table-barrier-ts":449571690058088458,"admin-job-type":0} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 + grep -q 'etcd info' {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:18:25 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:25.437 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00366ad80 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688965210246 MinTableBarrierTs:449571692202688513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 {"id":"e22ed8a8-de9e-4498-800f-b94e7d6ee3e8","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb37d e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688978055189,"min-table-barrier-ts":449571692202688513,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00366ad80 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688965210246 MinTableBarrierTs:449571692202688513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 {"id":"e22ed8a8-de9e-4498-800f-b94e7d6ee3e8","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb37d e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688978055189,"min-table-barrier-ts":449571692202688513,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7dbb39be-a55a-476d-b563-2ea64653043f {UpstreamID:7365782470846785413 Namespace:default ID:7dbb39be-a55a-476d-b563-2ea64653043f SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:18:18.031826409 +0800 CST StartTs:449571688152563713 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00366ad80 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571690380787721} {CheckpointTs:449571688965210246 MinTableBarrierTs:449571692202688513 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/511ee6a4-24b1-4a46-ada2-cc540514e34b {"id":"511ee6a4-24b1-4a46-ada2-cc540514e34b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 {"id":"e22ed8a8-de9e-4498-800f-b94e7d6ee3e8","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/e4089d28-b9b8-4324-a23c-8c7f86918df7 {"id":"e4089d28-b9b8-4324-a23c-8c7f86918df7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb313 e4089d28-b9b8-4324-a23c-8c7f86918df7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb353 511ee6a4-24b1-4a46-ada2-cc540514e34b /tidb/cdc/default/__cdc_meta__/owner/22318f4cc41fb37d e22ed8a8-de9e-4498-800f-b94e7d6ee3e8 /tidb/cdc/default/default/changefeed/info/7dbb39be-a55a-476d-b563-2ea64653043f {"upstream-id":7365782470846785413,"namespace":"default","changefeed-id":"7dbb39be-a55a-476d-b563-2ea64653043f","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:18:18.031826409+08:00","start-ts":449571688152563713,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571690380787721} /tidb/cdc/default/default/changefeed/status/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":449571688978055189,"min-table-barrier-ts":449571692202688513,"admin-job-type":0} /tidb/cdc/default/default/task/position/511ee6a4-24b1-4a46-ada2-cc540514e34b/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e22ed8a8-de9e-4498-800f-b94e7d6ee3e8/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved+ grep -q 'etcd info' -ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e4089d28-b9b8-4324-a23c-8c7f86918df7/7dbb39be-a55a-476d-b563-2ea64653043f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782470846785413 {"id":7365782470846785413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2024/05/06 15:18:25.930 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark12000] check diff successfully [2024/05/06 15:18:26.235 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] ./t/drop.test: ok! 48 test cases passed, take time 8.699121827 s time="2024-05-06T15:18:26+08:00" level=info msg="run test [drop] ok" time="2024-05-06T15:18:26+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000ca238 {{} {} 0} {0 0} [0xc0002ee000] map[] 0 1 0xc00003f080 false map[0xc0002ee000:map[0xc0002ee000:true]] map[] 0 0 0 0 0 0 0 0 0x4914e0}" wait process cdc.test exit for 1-th time... table move_table.usertable exists go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 "is-owner": true, run task successfully changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11959.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365782400118383158,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:18:26.268958337+08:00","start_ts":449571692508348420,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571692508348420,"checkpoint_ts":449571692508348420,"checkpoint_time":"2024-05-06 15:18:26.114"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5823.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331278080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:6703, start at 2024-05-06 15:18:25.031344533 +0800 CST m=+6.690556366 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:25.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:25.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:25.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331278080005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:6703, start at 2024-05-06 15:18:25.031344533 +0800 CST m=+6.690556366 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:25.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:25.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:25.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331278140008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-vhcbc, pid:6776, start at 2024-05-06 15:18:25.032960193 +0800 CST m=+6.637419907 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:25.040 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:25.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:25.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table processor_delay.t1 exists table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later table processor_delay.t14 exists go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 [2024/05/06 15:18:26.833 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] Create changefeed successfully! ID: 11fb9f24-d15f-446c-ba09-c25835054814 Info: {"upstream_id":7365782540811012596,"namespace":"default","id":"11fb9f24-d15f-446c-ba09-c25835054814","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:18:26.822199952+08:00","start_ts":449571692671139843,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571692671139843,"checkpoint_ts":449571692671139843,"checkpoint_time":"2024-05-06 15:18:26.735"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t15 not exists for 1-th check, retry later go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 wait process cdc.test exit for 2-th time... [2024/05/06 15:18:26.934 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:27.130 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 7-th check, retry later go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:27.450 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] table ddl_sequence.finish_mark exists check diff successfully go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 owner pid: 4082 owner id 964dc6d6-0b75-46fb-b5b0-cea892d69f42 [Mon May 6 15:18:27 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41854187.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:18:28 CST 2024] <<<<<< run test case region_merge success! >>>>>> [2024/05/06 15:18:28.248 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.047420372 s time="2024-05-06T15:18:27+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end + set +x wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later [2024/05/06 15:18:28.845 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later [2024/05/06 15:18:28.946 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 8-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8031.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:29 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/05/06 15:18:29.144 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/06 15:18:29.458 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark13000] [2024/05/06 15:18:29.463 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331293f80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:12674, start at 2024-05-06 15:18:26.857873678 +0800 CST m=+5.321811117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:26.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:26.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:26.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331293f80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:12674, start at 2024-05-06 15:18:26.857873678 +0800 CST m=+5.321811117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:26.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:26.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:26.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331293240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:12746, start at 2024-05-06 15:18:26.799217552 +0800 CST m=+5.213436320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:26.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:26.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:26.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 2-th check, retry later [2024/05/06 15:18:30.318 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2024/05/06 15:18:30.423 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark7000] [ts=449571693453115397] [2024/05/06 15:18:30.423 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark7000] [2024/05/06 15:18:30.423 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571693453115397] [2024/05/06 15:18:30.518 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:30.523 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table processor_resolved_ts_fallback.t1 exists [Mon May 6 15:18:30 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.59185920.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449571693258866690 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571693258866690 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:18:30 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.80638065.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:18:30.624 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:30.723 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:30.724 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark12000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 9-th check, retry later [2024/05/06 15:18:30.860 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:30.962 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table ddl_attributes.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:30 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002ccd680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571693465960455 MinTableBarrierTs:449571693465960455 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ad51998-3df0-40c2-8c63-66d583f1acf1 {"id":"2ad51998-3df0-40c2-8c63-66d583f1acf1","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f816b2 2ad51998-3df0-40c2-8c63-66d583f1acf1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571693465960455,"min-table-barrier-ts":449571693465960455,"admin-job-type":0} /tidb/cdc/default/default/task/position/2ad51998-3df0-40c2-8c63-66d583f1acf1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002ccd680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571693465960455 MinTableBarrierTs:449571693465960455 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ad51998-3df0-40c2-8c63-66d583f1acf1 {"id":"2ad51998-3df0-40c2-8c63-66d583f1acf1","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f816b2 2ad51998-3df0-40c2-8c63-66d583f1acf1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571693465960455,"min-table-barrier-ts":449571693465960455,"admin-job-type":0} /tidb/cdc/default/default/task/position/2ad51998-3df0-40c2-8c63-66d583f1acf1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002ccd680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571693465960455 MinTableBarrierTs:449571693465960455 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ad51998-3df0-40c2-8c63-66d583f1acf1 {"id":"2ad51998-3df0-40c2-8c63-66d583f1acf1","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/964dc6d6-0b75-46fb-b5b0-cea892d69f42 {"id":"964dc6d6-0b75-46fb-b5b0-cea892d69f42","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81680 964dc6d6-0b75-46fb-b5b0-cea892d69f42 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f816b2 2ad51998-3df0-40c2-8c63-66d583f1acf1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571693465960455,"min-table-barrier-ts":449571693465960455,"admin-job-type":0} /tidb/cdc/default/default/task/position/2ad51998-3df0-40c2-8c63-66d583f1acf1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/964dc6d6-0b75-46fb-b5b0-cea892d69f42/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert+ grep -q 'etcd info' -allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "964dc6d6-0b75-46fb-b5b0-cea892d69f42" | grep id + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571693399638022, "checkpoint_time": "2024-05-06 15:18:29.514", "error": null }' + echo '{ "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571693399638022, "checkpoint_time": "2024-05-06 15:18:29.514", "error": null }' { "upstream_id": 7365782400118383158, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449571693399638022, "checkpoint_time": "2024-05-06 15:18:29.514", "error": null } ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571693399638022, '"checkpoint_time":' '"2024-05-06' '15:18:29.514",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365782400118383158, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449571693399638022, '"checkpoint_time":' '"2024-05-06' '15:18:29.514",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] [2024/05/06 15:18:31.158 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later [2024/05/06 15:18:31.529 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12045.out cli unsafe delete-service-gc-safepoint [Mon May 6 15:18:31 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1401414016.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... "cluster-id": "default" "id": "2ad51998-3df0-40c2-8c63-66d583f1acf1", "cluster-id": "default" run task successfully table mark.finish_mark_3 not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:32.724 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:18:32.933 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] table test.finish_mark not exists for 3-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later [2024/05/06 15:18:33.034 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/05/06 15:18:33.237 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12068.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 [2024/05/06 15:18:33.618 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/11fb9f24-d15f-446c-ba09-c25835054814 {UpstreamID:7365782540811012596 Namespace:default ID:11fb9f24-d15f-446c-ba09-c25835054814 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:26.822199952 +0800 CST StartTs:449571692671139843 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033be630 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571692684247046} {CheckpointTs:449571694335754244 MinTableBarrierTs:449571694335754246 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1738cfb7-000e-4279-bd61-abffa77fddf1 {"id":"1738cfb7-000e-4279-bd61-abffa77fddf1","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eee50 1738cfb7-000e-4279-bd61-abffa77fddf1 /tidb/cdc/default/default/changefeed/info/11fb9f24-d15f-446c-ba09-c25835054814 {"upstream-id":7365782540811012596,"namespace":"default","changefeed-id":"11fb9f24-d15f-446c-ba09-c25835054814","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:26.822199952+08:00","start-ts":449571692671139843,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571692684247046} /tidb/cdc/default/default/changefeed/status/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":449571694335754244,"min-table-barrier-ts":449571694335754246,"admin-job-type":0} /tidb/cdc/default/default/task/position/1738cfb7-000e-4279-bd61-abffa77fddf1/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8db376b-b4ac-4722-94ff-ccf507de1cd8/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/11fb9f24-d15f-446c-ba09-c25835054814 {UpstreamID:7365782540811012596 Namespace:default ID:11fb9f24-d15f-446c-ba09-c25835054814 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:26.822199952 +0800 CST StartTs:449571692671139843 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033be630 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571692684247046} {CheckpointTs:449571694335754244 MinTableBarrierTs:449571694335754246 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1738cfb7-000e-4279-bd61-abffa77fddf1 {"id":"1738cfb7-000e-4279-bd61-abffa77fddf1","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eee50 1738cfb7-000e-4279-bd61-abffa77fddf1 /tidb/cdc/default/default/changefeed/info/11fb9f24-d15f-446c-ba09-c25835054814 {"upstream-id":7365782540811012596,"namespace":"default","changefeed-id":"11fb9f24-d15f-446c-ba09-c25835054814","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:26.822199952+08:00","start-ts":449571692671139843,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571692684247046} /tidb/cdc/default/default/changefeed/status/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":449571694335754244,"min-table-barrier-ts":449571694335754246,"admin-job-type":0} /tidb/cdc/default/default/task/position/1738cfb7-000e-4279-bd61-abffa77fddf1/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8db376b-b4ac-4722-94ff-ccf507de1cd8/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/11fb9f24-d15f-446c-ba09-c25835054814 {UpstreamID:7365782540811012596 Namespace:default ID:11fb9f24-d15f-446c-ba09-c25835054814 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:26.822199952 +0800 CST StartTs:449571692671139843 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033be630 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571692684247046} {CheckpointTs:449571694335754244 MinTableBarrierTs:449571694335754246 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1738cfb7-000e-4279-bd61-abffa77fddf1 {"id":"1738cfb7-000e-4279-bd61-abffa77fddf1","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/f8db376b-b4ac-4722-94ff-ccf507de1cd8 {"id":"f8db376b-b4ac-4722-94ff-ccf507de1cd8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eedf4 f8db376b-b4ac-4722-94ff-ccf507de1cd8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc45eee50 1738cfb7-000e-4279-bd61-abffa77fddf1 /tidb/cdc/default/default/changefeed/info/11fb9f24-d15f-446c-ba09-c25835054814 {"upstream-id":7365782540811012596,"namespace":"default","changefeed-id":"11fb9f24-d15f-446c-ba09-c25835054814","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:26.822199952+08:00","start-ts":449571692671139843,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571692684247046} /tidb/cdc/default/default/changefeed/status/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":449571694335754244,"min-table-barrier-ts":449571694335754246,"admin-job-type":0} /tidb/cdc/default/default/task/position/1738cfb7-000e-4279-bd61-abffa77fddf1/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f8db376b-b4ac-4722-94ff-ccf507de1cd8/11fb9f24-d15f-446c-ba09-c25835054814 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782540811012596 {"id":7365782540811012596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 {"id":"56acaa2e-5db9-4ccb-8e8f-aa3e4085de49","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc46b64f6 56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 /tidb/cdc/default/default/upstream/7365782560778829511 {"id":7365782560778829511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 {"id":"56acaa2e-5db9-4ccb-8e8f-aa3e4085de49","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc46b64f6 56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 /tidb/cdc/default/default/upstream/7365782560778829511 {"id":7365782560778829511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 {"id":"56acaa2e-5db9-4ccb-8e8f-aa3e4085de49","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc46b64f6 56acaa2e-5db9-4ccb-8e8f-aa3e4085de49 /tidb/cdc/default/default/upstream/7365782560778829511 {"id":7365782560778829511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8116.out cli changefeed create --start-ts=449571693258866690 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... capture_id: 2ad51998-3df0-40c2-8c63-66d583f1acf1 Create changefeed successfully! ID: fb352adc-98db-40ab-b69c-7f352e8b6049 Info: {"upstream_id":7365782560778829511,"namespace":"default","id":"fb352adc-98db-40ab-b69c-7f352e8b6049","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:18:34.042559801+08:00","start_ts":449571693258866690,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571693258866690,"checkpoint_ts":449571693258866690,"checkpoint_time":"2024-05-06 15:18:28.977"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:18:34.024 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark14000] table mark.finish_mark_3 not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:34.738 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] table test.finish_mark not exists for 4-th check, retry later table ddl_attributes.finish_mark not exists for 5-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:32 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> [2024/05/06 15:18:34.946 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:35.048 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5d98dc-b68b-44cc-b8d0-6f08da31d377 {"id":"7f5d98dc-b68b-44cc-b8d0-6f08da31d377","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751df6 7f5d98dc-b68b-44cc-b8d0-6f08da31d377 /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5d98dc-b68b-44cc-b8d0-6f08da31d377 {"id":"7f5d98dc-b68b-44cc-b8d0-6f08da31d377","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751df6 7f5d98dc-b68b-44cc-b8d0-6f08da31d377 /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f5d98dc-b68b-44cc-b8d0-6f08da31d377 {"id":"7f5d98dc-b68b-44cc-b8d0-6f08da31d377","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751df6 7f5d98dc-b68b-44cc-b8d0-6f08da31d377 /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 15:18:35.251 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] + set +x + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 [2024/05/06 15:18:35.633 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later go: downloading golang.org/x/text v0.14.0 ***************** properties ***************** "fieldlengthdistribution"="constant" "mysql.port"="4000" "threadcount"="10" "mysql.db"="consistent_replicate_storage_file_large_value" "requestdistribution"="uniform" "fieldlength"="15000" "updateproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "workload"="core" "insertproportion"="0" "readallfields"="true" "recordcount"="100" "scanproportion"="0" "fieldcount"="1000" "readproportion"="0" "operationcount"="0" "mysql.user"="root" ********************************************** table test.finish_mark not exists for 5-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later [2024/05/06 15:18:36.752 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:18:36.961 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later [2024/05/06 15:18:37.060 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:37.262 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] table mark.finish_mark_3 not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:18:37.645 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 5779 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later [2024/05/06 15:18:38.243 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark15000] go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/stretchr/testify v1.9.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.162.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 table test.finish_mark not exists for 6-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12157.out cli unsafe resolve-lock --region=78 PASS nonempty select id, val from test.availability1 where id=1 and val=1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later [2024/05/06 15:18:38.763 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] coverage: 2.0% of statements in github.com/pingcap/tiflow/... run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 13-th check, retry later [2024/05/06 15:18:39.025 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:39.072 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible table ddl_attributes.finish_mark not exists for 7-th check, retry later [2024/05/06 15:18:39.322 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 [2024/05/06 15:18:39.657 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2024/05/06 15:18:39.672 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark8000] [ts=449571695569666053] [2024/05/06 15:18:39.672 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark8000] [2024/05/06 15:18:39.672 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571695569666053] [2024/05/06 15:18:39.733 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:39.737 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:39.742 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:39.748 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:39.749 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark13000] go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12184.out cli unsafe resolve-lock --region=78 --ts=449571694920073221 table test.finish_mark not exists for 7-th check, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:18:40.776 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] table mark.finish_mark_3 not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:41.043 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:41.086 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table ddl_attributes.finish_mark not exists for 8-th check, retry later [2024/05/06 15:18:41.342 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x [2024/05/06 15:18:41.749 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 131k 3819 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:18:42.839 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] table mark.finish_mark_3 not exists for 15-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table ddl_attributes.finish_mark not exists for 9-th check, retry later [2024/05/06 15:18:43.056 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:43.118 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/felixge/httpsnoop v1.0.4 empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:18:43.355 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] go: downloading github.com/jmespath/go-jmespath v0.4.0 [2024/05/06 15:18:43.820 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 15:18:43.844 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark16000] go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table mark.finish_mark_3 not exists for 16-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:44.850 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_attributes.finish_mark exists check diff successfully [2024/05/06 15:18:45.121 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:45.136 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:18:45.421 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] [2024/05/06 15:18:45.437 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark9000] [ts=449571697542299653] [2024/05/06 15:18:45.437 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark9000] [2024/05/06 15:18:45.437 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571697542299653] [2024/05/06 15:18:45.439 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:45.442 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 154k 0 --:--:-- --:--:-- --:--:-- 171k { "version": "v7.5.1-23-gf21441d0a", "git_hash": "f21441d0a8b3471f8548ffca9a37735214c532e0", "id": "e2df86f0-e68f-4ca9-8efb-1f99c0e25e65", "pid": 11430, "is_owner": true }wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... [2024/05/06 15:18:45.446 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:45.451 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:45.451 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark14000] [2024/05/06 15:18:45.839 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 15:18:40.075 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 6"] wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... Run finished, takes 9.693451963s INSERT_ERROR - Takes(s): 8.6, Count: 100, OPS: 11.6, Avg(us): 329439, Min(us): 194826, Max(us): 776269, 95th(us): 593000, 99th(us): 637000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:46 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table test.finish_mark not exists for 10-th check, retry later table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:18:47 CST 2024] <<<<<< run test case cli success! >>>>>> table mark.finish_mark_3 not exists for 17-th check, retry later [2024/05/06 15:18:46.917 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:18:47.133 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:47.217 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:47.451 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 15:18:47.935 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark17000] [2024/05/06 15:18:48.018 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later [2024/05/06 15:18:48.937 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3313d3340010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:6478, start at 2024-05-06 15:18:47.269465556 +0800 CST m=+26.524591878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:47.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:47.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3313d3340010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:6478, start at 2024-05-06 15:18:47.269465556 +0800 CST m=+26.524591878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:47.277 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:47.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:47.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3312a5d80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-2rgsm, pid:6562, start at 2024-05-06 15:18:28.004449825 +0800 CST m=+7.210877860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:28.011 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:28.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:28.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:18:49.220 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:49.233 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:49.530 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:18:50.035 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 12-th check, retry later table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:50.950 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:18:51.028 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark18000] [Mon May 6 15:18:51 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1917719179.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:18:51.232 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:51.247 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] table mark.finish_mark_3 not exists for 18-th check, retry later table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later [2024/05/06 15:18:51.542 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later [2024/05/06 15:18:52.047 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331413a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:6378, start at 2024-05-06 15:18:51.401306074 +0800 CST m=+5.284165797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:51.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:51.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:51.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.finish_mark not exists for 13-th check, retry later [2024/05/06 15:18:52.962 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later [2024/05/06 15:18:53.244 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2024/05/06 15:18:53.260 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:18:53.421 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark10000] [ts=449571699324878853] [2024/05/06 15:18:53.421 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark10000] [2024/05/06 15:18:53.421 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571699324878853] [2024/05/06 15:18:53.425 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:53.431 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:53.437 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:53.523 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:53.524 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark15000] [2024/05/06 15:18:53.554 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] table mark.finish_mark_3 not exists for 19-th check, retry later wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later [2024/05/06 15:18:54.118 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143b18000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:9941, start at 2024-05-06 15:18:53.906437329 +0800 CST m=+5.784257282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:53.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:53.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:53.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:18:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9144c07e-06aa-4765-a4bb-540e95fa960d {"id":"9144c07e-06aa-4765-a4bb-540e95fa960d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4795a15 9144c07e-06aa-4765-a4bb-540e95fa960d /tidb/cdc/default/default/upstream/7365782573994459961 {"id":7365782573994459961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9144c07e-06aa-4765-a4bb-540e95fa960d {"id":"9144c07e-06aa-4765-a4bb-540e95fa960d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4795a15 9144c07e-06aa-4765-a4bb-540e95fa960d /tidb/cdc/default/default/upstream/7365782573994459961 {"id":7365782573994459961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9144c07e-06aa-4765-a4bb-540e95fa960d {"id":"9144c07e-06aa-4765-a4bb-540e95fa960d","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4795a15 9144c07e-06aa-4765-a4bb-540e95fa960d /tidb/cdc/default/default/upstream/7365782573994459961 {"id":7365782573994459961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.19446.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:18:54 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: e0d4f211-d67b-4542-8711-d94726ce6283 Info: {"upstream_id":7365782573994459961,"namespace":"default","id":"e0d4f211-d67b-4542-8711-d94726ce6283","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:18:54.661044004+08:00","start_ts":449571699961626625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571699961626625,"checkpoint_ts":449571699961626625,"checkpoint_time":"2024-05-06 15:18:54.546"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:18:55.033 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:18:55.140 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark19000] table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later [2024/05/06 15:18:51.498 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 15:18:51.502 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 15:18:51.502 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=e4089d28-b9b8-4324-a23c-8c7f86918df7] [2024/05/06 15:18:51.503 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 15:18:51.503 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 15:18:51.679 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=511ee6a4-24b1-4a46-ada2-cc540514e34b] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 15:18:51.878 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 15:18:52.078 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=e4089d28-b9b8-4324-a23c-8c7f86918df7] [processorDetail="{\"table_ids\":[104,106]}"] [2024/05/06 15:18:52.078 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"511ee6a4-24b1-4a46-ada2-cc540514e34b\":[],\"e22ed8a8-de9e-4498-800f-b94e7d6ee3e8\":[],\"e4089d28-b9b8-4324-a23c-8c7f86918df7\":[{\"ID\":104,\"Changefeed\":\"7dbb39be-a55a-476d-b563-2ea64653043f\"},{\"ID\":106,\"Changefeed\":\"7dbb39be-a55a-476d-b563-2ea64653043f\"}]}"] [2024/05/06 15:18:52.078 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=7dbb39be-a55a-476d-b563-2ea64653043f&target-cp-id=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8&table-id=104"] [2024/05/06 15:18:52.128 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=104] [2024/05/06 15:18:52.128 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=7dbb39be-a55a-476d-b563-2ea64653043f&target-cp-id=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8&table-id=106"] [2024/05/06 15:18:52.178 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/06 15:18:52.178 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=e4089d28-b9b8-4324-a23c-8c7f86918df7] [targetCapture=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8] table move_table.check1 exists check diff successfully empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass [2024/05/06 15:18:55.338 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143e880017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:5839, start at 2024-05-06 15:18:54.161954518 +0800 CST m=+7.185159349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:54.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:54.163 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:54.163 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143e880017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:5839, start at 2024-05-06 15:18:54.161954518 +0800 CST m=+7.185159349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:54.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:54.163 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:54.163 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143f00000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-nn55c, pid:5866, start at 2024-05-06 15:18:54.168817318 +0800 CST m=+7.085845947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:54.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:54.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:54.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log table test.finish_mark not exists for 14-th check, retry later [2024/05/06 15:18:55.524 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:18:55.633 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] table mark.finish_mark_3 not exists for 20-th check, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331413a40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:6378, start at 2024-05-06 15:18:51.401306074 +0800 CST m=+5.284165797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:51.407 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:51.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:51.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331414540010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-6x3z6, pid:6461, start at 2024-05-06 15:18:51.430075882 +0800 CST m=+5.224178426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:51.437 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:51.413 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:51.413 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [2024/05/06 15:18:56.132 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331454780004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:6378, start at 2024-05-06 15:18:55.519349243 +0800 CST m=+5.216537774 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:55.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:55.518 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:55.518 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331454780004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:6378, start at 2024-05-06 15:18:55.519349243 +0800 CST m=+5.216537774 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:55.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:55.518 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:55.518 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331453940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:6464, start at 2024-05-06 15:18:55.491999241 +0800 CST m=+5.110888874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:55.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:55.461 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:55.461 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143b18000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:9941, start at 2024-05-06 15:18:53.906437329 +0800 CST m=+5.784257282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:53.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:53.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:53.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33143c700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qh84h, pid:10024, start at 2024-05-06 15:18:54.012901433 +0800 CST m=+5.837708323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:20:54.024 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:18:54.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:08:54.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Mon May 6 15:18:56 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.45774579.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later [2024/05/06 15:18:57.084 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 15:18:57.086 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 15:18:57.087 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=e4089d28-b9b8-4324-a23c-8c7f86918df7] [2024/05/06 15:18:57.087 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 15:18:57.087 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 15:18:57.045 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:18:57.278 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=511ee6a4-24b1-4a46-ada2-cc540514e34b] [processorDetail="{\"table_ids\":[]}"] ***************** properties ***************** "operationcount"="0" "updateproportion"="0" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_rocks" "readallfields"="true" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="1000" "insertproportion"="0" "table"="a1" ********************************************** [2024/05/06 15:18:57.351 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:57.478 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8] [processorDetail="{\"table_ids\":[106]}"] [2024/05/06 15:18:57.678 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=7dbb39be-a55a-476d-b563-2ea64653043f] [captureID=e4089d28-b9b8-4324-a23c-8c7f86918df7] [processorDetail="{\"table_ids\":[108]}"] [2024/05/06 15:18:57.678 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"511ee6a4-24b1-4a46-ada2-cc540514e34b\":[],\"e22ed8a8-de9e-4498-800f-b94e7d6ee3e8\":[{\"ID\":106,\"Changefeed\":\"7dbb39be-a55a-476d-b563-2ea64653043f\"}],\"e4089d28-b9b8-4324-a23c-8c7f86918df7\":[{\"ID\":108,\"Changefeed\":\"7dbb39be-a55a-476d-b563-2ea64653043f\"}]}"] [2024/05/06 15:18:57.678 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=7dbb39be-a55a-476d-b563-2ea64653043f&target-cp-id=511ee6a4-24b1-4a46-ada2-cc540514e34b&table-id=106"] [2024/05/06 15:18:57.537 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:18:57.641 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20001] [2024/05/06 15:18:57.642 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20002] [2024/05/06 15:18:57.643 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20003] [2024/05/06 15:18:57.645 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 15:18:57.646 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] table mark.finish_mark_3 not exists for 21-th check, retry later Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [2024/05/06 15:18:57.728 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/06 15:18:57.728 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=e22ed8a8-de9e-4498-800f-b94e7d6ee3e8] [targetCapture=511ee6a4-24b1-4a46-ada2-cc540514e34b] check diff successfully Run finished, takes 619.943797ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1646.6, Avg(us): 1204, Min(us): 927, Max(us): 12610, 95th(us): 2000, 99th(us): 2000 table test.finish_mark not exists for 15-th check, retry later ***************** properties ***************** "operationcount"="0" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "threadcount"="2" "mysql.user"="root" "updateproportion"="0" "scanproportion"="0" "recordcount"="1000" "dotransactions"="false" "insertproportion"="0" "readproportion"="0" "mysql.port"="4000" "table"="a2" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" ********************************************** [Pipeline] // cache [2024/05/06 15:18:58.144 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 15:18:58.151 +08:00] [INFO] [case.go:539] ["upstream create table executed"] [tblName=finishmark20000] [2024/05/06 15:18:58.151 +08:00] [INFO] [case.go:556] ["one upstream workload finished"] [round=20004] table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7792.out cli tso query --pd=http://127.0.0.1:2379 table move_table.check2 not exists for 1-th check, retry later [Pipeline] // dir + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11284.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2024/05/06 15:18:59.058 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] Run finished, takes 587.619296ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1741.1, Avg(us): 1145, Min(us): 855, Max(us): 13180, 95th(us): 2000, 99th(us): 2000 [Mon May 6 15:18:58 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.77047706.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv [Pipeline] } [2024/05/06 15:18:59.420 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2024/05/06 15:18:59.446 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark11000] [ts=449571700943618053] [2024/05/06 15:18:59.446 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark11000] [2024/05/06 15:18:59.446 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571700943618053] [2024/05/06 15:18:59.449 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:18:59.451 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:59.456 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] ***************** properties ***************** "mysql.port"="4000" "table"="a3" "insertproportion"="0" "readallfields"="true" "workload"="core" "dotransactions"="false" "recordcount"="1000" "mysql.host"="127.0.0.1" "updateproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "threadcount"="2" "mysql.user"="root" "scanproportion"="0" "readproportion"="0" "mysql.db"="multi_rocks" ********************************************** =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // node [2024/05/06 15:18:59.520 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:18:59.520 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark16000] [2024/05/06 15:18:59.550 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:18:59.657 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] + set +x + tso='449571700924481537 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571700924481537 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:18:59 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.78217823.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Mon May 6 15:18:59 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.70817083.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 16-th check, retry later [Pipeline] // podTemplate Run finished, takes 613.289036ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1679.3, Avg(us): 1195, Min(us): 851, Max(us): 17706, 95th(us): 2000, 99th(us): 2000 [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2024/05/06 15:19:00.157 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] table mark.finish_mark_3 not exists for 22-th check, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later [Pipeline] // stage table move_table.check2 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ddb00 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571700032667649 MinTableBarrierTs:449571701343387650 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571700032667649,"min-table-barrier-ts":449571701343387650,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ddb00 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571700032667649 MinTableBarrierTs:449571701343387650 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571700032667649,"min-table-barrier-ts":449571701343387650,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/73657821+ grep -q 'failed to get info:' 49811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014ddb00 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571700032667649 MinTableBarrierTs:449571701343387650 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571700032667649, checkpointTs: 449571700032667649, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571700032667649,"min-table-barrier-ts":449571701343387650,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Pipeline] } ***************** properties ***************** "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.port"="4000" "requestdistribution"="uniform" "recordcount"="1000" "workload"="core" "scanproportion"="0" "table"="a4" "operationcount"="0" "mysql.db"="multi_rocks" "insertproportion"="0" "updateproportion"="0" "dotransactions"="false" "mysql.user"="root" "readproportion"="0" "readallfields"="true" ********************************************** + set +x + tso='449571701062369284 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571701062369284 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:19:00 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1131611318.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Run finished, takes 611.857536ms INSERT - Takes(s): 0.6, Count: 999, OPS: 1685.0, Avg(us): 1174, Min(us): 864, Max(us): 18921, 95th(us): 2000, 99th(us): 2000 [2024/05/06 15:19:01.133 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ***************** properties ***************** "mysql.port"="4000" "requestdistribution"="uniform" "readallfields"="true" "insertproportion"="0" "operationcount"="0" "recordcount"="1000" "threadcount"="2" "table"="a5" "readproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="multi_rocks" "mysql.user"="root" "workload"="core" "updateproportion"="0" "scanproportion"="0" ********************************************** [2024/05/06 15:19:01.521 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 15:19:01.625 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:19:01.721 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] wait process cdc.test exit for 2-th time... "is-owner": true, run task successfully table test.finish_mark not exists for 17-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 {"id":"d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae7ef d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 {"id":"d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae7ef d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 {"id":"d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae7ef d4d8effe-630a-4a7a-bdcb-6e3bdf7cb4d9 /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7756.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table wait process cdc.test exit for 3-th time... table mark.finish_mark_3 not exists for 23-th check, retry later Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365782699936505141,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T15:19:02.160997837+08:00","start_ts":449571701933473793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571701933473793,"checkpoint_ts":449571701933473793,"checkpoint_time":"2024-05-06 15:19:02.068"} PASS Run finished, takes 592.232408ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1812.3, Avg(us): 1152, Min(us): 805, Max(us): 40385, 95th(us): 2000, 99th(us): 2000 table consistent_replicate_storage_file_large_value.t2 not exists for 7-th check, retry later [2024/05/06 15:19:02.217 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:19:02 CST 2024] <<<<<< run test case move_table success! >>>>>> owner pid: 4582 owner id 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a default owner key /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 [Mon May 6 15:19:02 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.46924694.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dca52d48-6924-42e1-9d5a-b69159e12227 {"id":"dca52d48-6924-42e1-9d5a-b69159e12227","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4d985f8 dca52d48-6924-42e1-9d5a-b69159e12227 /tidb/cdc/default/default/upstream/7365782679486119657 {"id":7365782679486119657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dca52d48-6924-42e1-9d5a-b69159e12227 {"id":"dca52d48-6924-42e1-9d5a-b69159e12227","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4d985f8 dca52d48-6924-42e1-9d5a-b69159e12227 /tidb/cdc/default/default/upstream/7365782679486119657 {"id":7365782679486119657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dca52d48-6924-42e1-9d5a-b69159e12227 {"id":"dca52d48-6924-42e1-9d5a-b69159e12227","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4d985f8 dca52d48-6924-42e1-9d5a-b69159e12227 /tidb/cdc/default/default/upstream/7365782679486119657 {"id":7365782679486119657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7868.out cli changefeed create --start-ts=449571700924481537 --sink-uri=mysql://root@127.0.0.1:3306/ + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e6f104-2c5d-4363-9343-3a987db7a939 {"id":"48e6f104-2c5d-4363-9343-3a987db7a939","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df86f5 48e6f104-2c5d-4363-9343-3a987db7a939 /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e6f104-2c5d-4363-9343-3a987db7a939 {"id":"48e6f104-2c5d-4363-9343-3a987db7a939","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df86f5 48e6f104-2c5d-4363-9343-3a987db7a939 /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48e6f104-2c5d-4363-9343-3a987db7a939 {"id":"48e6f104-2c5d-4363-9343-3a987db7a939","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df86f5 48e6f104-2c5d-4363-9343-3a987db7a939 /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/05/06 15:19:03.146 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2024/05/06 15:19:03.228 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark12000] [ts=449571701900443656] [2024/05/06 15:19:03.228 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark12000] [2024/05/06 15:19:03.228 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571701900443656] [2024/05/06 15:19:03.230 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:03.233 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:03.238 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:03.243 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:03.243 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Create changefeed successfully! ID: 99471aeb-bf9e-4e5b-8d58-893df596a4d8 Info: {"upstream_id":7365782679486119657,"namespace":"default","id":"99471aeb-bf9e-4e5b-8d58-893df596a4d8","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T15:19:03.100598348+08:00","start_ts":449571700924481537,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571700924481537,"checkpoint_ts":449571700924481537,"checkpoint_time":"2024-05-06 15:18:58.219"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t32 exists [2024/05/06 15:19:03.533 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] table processor_delay.t33 not exists for 1-th check, retry later * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68023e81-22dd-4f8e-a01d-2a881ef9e42f {"id":"68023e81-22dd-4f8e-a01d-2a881ef9e42f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4dfaaf6 68023e81-22dd-4f8e-a01d-2a881ef9e42f /tidb/cdc/default/default/upstream/7365782685371456517 {"id":7365782685371456517,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68023e81-22dd-4f8e-a01d-2a881ef9e42f {"id":"68023e81-22dd-4f8e-a01d-2a881ef9e42f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4dfaaf6 68023e81-22dd-4f8e-a01d-2a881ef9e42f /tidb/cdc/default/default/upstream/7365782685371456517 {"id":7365782685371456517,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68023e81-22dd-4f8e-a01d-2a881ef9e42f {"id":"68023e81-22dd-4f8e-a01d-2a881ef9e42f","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4dfaaf6 68023e81-22dd-4f8e-a01d-2a881ef9e42f /tidb/cdc/default/default/upstream/7365782685371456517 {"id":7365782685371456517,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11371.out cli changefeed create --start-ts=449571701062369284 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table test.finish_mark not exists for 18-th check, retry later + set +x Create changefeed successfully! ID: 773b01e5-acf4-41be-a5cd-657ac8bf658f Info: {"upstream_id":7365782685371456517,"namespace":"default","id":"773b01e5-acf4-41be-a5cd-657ac8bf658f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:19:03.699464272+08:00","start_ts":449571701062369284,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571701062369284,"checkpoint_ts":449571701062369284,"checkpoint_time":"2024-05-06 15:18:58.745"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:19:03.637 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:19:03.735 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] table mark.finish_mark_3 not exists for 24-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later [2024/05/06 15:19:04.232 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] table consistent_replicate_storage_file_large_value.t2 exists table multi_rocks.finish_mark not exists for 1-th check, retry later check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) table test.table1 not exists for 1-th check, retry later [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } + set +x [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 1-th time... table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 2-th time... + set +x [Pipeline] // withEnv [Pipeline] } [2024/05/06 15:19:05.545 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [Pipeline] // node [Pipeline] } skip some SQLs in tidb v4.0.x table common_1.v1 not exists for 1-th check, retry later [Pipeline] // podTemplate Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } [2024/05/06 15:19:05.650 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:19:05.748 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:05 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1428914291.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table resourcecontrol.finish_mark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 25-th check, retry later [Pipeline] // withEnv + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003663290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571702654107652 MinTableBarrierTs:449571702654107652 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/61456fe6-b202-43f6-b2e5-581100224218 {"id":"61456fe6-b202-43f6-b2e5-581100224218","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8177b 61456fe6-b202-43f6-b2e5-581100224218 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571702654107652,"min-table-barrier-ts":449571702654107652,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/61456fe6-b202-43f6-b2e5-581100224218/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003663290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571702654107652 MinTableBarrierTs:449571702654107652 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/61456fe6-b202-43f6-b2e5-581100224218 {"id":"61456fe6-b202-43f6-b2e5-581100224218","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8177b 61456fe6-b202-43f6-b2e5-581100224218 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571702654107652,"min-table-barrier-ts":449571702654107652,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/61456fe6-b202-43f6-b2e5-581100224218/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert+ grep -q 'failed to get info:' -allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003663290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571702654107652 MinTableBarrierTs:449571702654107652 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a {"id":"5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/61456fe6-b202-43f6-b2e5-581100224218 {"id":"61456fe6-b202-43f6-b2e5-581100224218","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81748 5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8177b 61456fe6-b202-43f6-b2e5-581100224218 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571702654107652,"min-table-barrier-ts":449571702654107652,"admin-job-type":0} /tidb/cdc/default/default/task/position/5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/61456fe6-b202-43f6-b2e5-581100224218/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "5f46a0c6-9426-49d9-94a4-5f1c71f2ba6a default" | grep id [Pipeline] } table multi_rocks.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 19-th check, retry later [Pipeline] // stage [Pipeline] } [2024/05/06 15:19:06.317 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2024/05/06 15:19:06.419 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark13000] [ts=449571702844162059] [2024/05/06 15:19:06.419 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark13000] [2024/05/06 15:19:06.419 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571702844162059] [2024/05/06 15:19:06.425 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:06.430 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:06.435 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] table test.table1 exists table test.table2 not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later [2024/05/06 15:19:06.522 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:06.522 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark17000] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "id": "61456fe6-b202-43f6-b2e5-581100224218", run task successfully table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later [2024/05/06 15:19:07.558 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] table resourcecontrol.finish_mark exists check diff successfully table mark.finish_mark_3 not exists for 26-th check, retry later [2024/05/06 15:19:07.663 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:19:07.822 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] table multi_rocks.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 20-th check, retry later table common_1.v1 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... table test.table2 exists table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later check diff successfully capture_id: 61456fe6-b202-43f6-b2e5-581100224218 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '61456fe6-b202-43f6-b2e5-581100224218' 61456fe6-b202-43f6-b2e5-581100224218 run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 22515 0 --:--:-- --:--:-- --:--:-- 25142 "is_owner": false run task successfully [2024/05/06 15:19:08.523 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3d83880a-de7b-4723-9f25-7005399774cf {UpstreamID:7365782570610106361 Namespace:default ID:3d83880a-de7b-4723-9f25-7005399774cf SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:35.176817903 +0800 CST StartTs:449571694841430021 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004894510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571694880751619} {CheckpointTs:449571698537660442 MinTableBarrierTs:449571703439753224 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45f8a44e-4679-407d-b9e1-d7c039788943 {"id":"45f8a44e-4679-407d-b9e1-d7c039788943","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751eda 45f8a44e-4679-407d-b9e1-d7c039788943 /tidb/cdc/default/default/changefeed/info/3d83880a-de7b-4723-9f25-7005399774cf {"upstream-id":7365782570610106361,"namespace":"default","changefeed-id":"3d83880a-de7b-4723-9f25-7005399774cf","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:35.176817903+08:00","start-ts":449571694841430021,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571694880751619} /tidb/cdc/default/default/changefeed/status/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":449571698537660442,"min-table-barrier-ts":449571703439753224,"admin-job-type":0} /tidb/cdc/default/default/task/position/45f8a44e-4679-407d-b9e1-d7c039788943/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3d83880a-de7b-4723-9f25-7005399774cf {UpstreamID:7365782570610106361 Namespace:default ID:3d83880a-de7b-4723-9f25-7005399774cf SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:35.176817903 +0800 CST StartTs:449571694841430021 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004894510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571694880751619} {CheckpointTs:449571698537660442 MinTableBarrierTs:449571703439753224 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45f8a44e-4679-407d-b9e1-d7c039788943 {"id":"45f8a44e-4679-407d-b9e1-d7c039788943","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751eda 45f8a44e-4679-407d-b9e1-d7c039788943 /tidb/cdc/default/default/changefeed/info/3d83880a-de7b-4723-9f25-7005399774cf {"upstream-id":7365782570610106361,"namespace":"default","changefeed-id":"3d83880a-de7b-4723-9f25-7005399774cf","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:35.176817903+08:00","start-ts":449571694841430021,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571694880751619} /tidb/cdc/default/default/changefeed/status/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":449571698537660442,"min-table-barrier-ts":449571703439753224,"admin-job-type":0} /tidb/cdc/default/default/task/position/45f8a44e-4679-407d-b9e1-d7c039788943/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3d83880a-de7b-4723-9f25-7005399774cf {UpstreamID:7365782570610106361 Namespace:default ID:3d83880a-de7b-4723-9f25-7005399774cf SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:18:35.176817903 +0800 CST StartTs:449571694841430021 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004894510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571694880751619} {CheckpointTs:449571698537660442 MinTableBarrierTs:449571703439753224 AdminJobType:noop} span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571698537660442, checkpointTs: 449571698537660442, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45f8a44e-4679-407d-b9e1-d7c039788943 {"id":"45f8a44e-4679-407d-b9e1-d7c039788943","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4751eda 45f8a44e-4679-407d-b9e1-d7c039788943 /tidb/cdc/default/default/changefeed/info/3d83880a-de7b-4723-9f25-7005399774cf {"upstream-id":7365782570610106361,"namespace":"default","changefeed-id":"3d83880a-de7b-4723-9f25-7005399774cf","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:18:35.176817903+08:00","start-ts":449571694841430021,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571694880751619} /tidb/cdc/default/default/changefeed/status/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":449571698537660442,"min-table-barrier-ts":449571703439753224,"admin-job-type":0} /tidb/cdc/default/default/task/position/45f8a44e-4679-407d-b9e1-d7c039788943/3d83880a-de7b-4723-9f25-7005399774cf {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782570610106361 {"id":7365782570610106361,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:09 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:19:09.570 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 15:19:09.676 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] table multi_rocks.finish_mark not exists for 4-th check, retry later [2024/05/06 15:19:09.837 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2024/05/06 15:19:09.937 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark14000] [ts=449571703800987653] [2024/05/06 15:19:09.937 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark14000] [2024/05/06 15:19:09.937 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571703800987653] [2024/05/06 15:19:10.021 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] table test.finish_mark not exists for 21-th check, retry later table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later [2024/05/06 15:19:10.025 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:10.031 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:10.119 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:10.119 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] table mark.finish_mark_3 not exists for 27-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later [2024/05/06 15:19:10.537 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 22-th check, retry later [2024/05/06 15:19:11.634 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 15:19:11.818 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] Resume changefeed with checkpointTs3 449571704108220418 table common_1.recover_and_insert not exists for 2-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 28-th check, retry later table test.table5 not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark exists table multi_rocks.finish_mark not exists for 5-th check, retry later wait process 7709 exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:19:12.556 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] wait process 7709 exit for 2-th time... table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7709) - No such process wait process 7709 exit for 3-th time... process 7709 already exit [Mon May 6 15:19:13 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.78427844.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:19:13.717 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] table test.finish_mark not exists for 23-th check, retry later table common_1.recover_and_insert not exists for 3-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2024/05/06 15:19:13.837 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2024/05/06 15:19:13.935 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark15000] [ts=449571704784027653] [2024/05/06 15:19:13.935 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark15000] [2024/05/06 15:19:13.935 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571704784027653] [2024/05/06 15:19:13.939 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:13.942 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:13.948 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:14.020 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:14.021 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark18000] table mark.finish_mark_3 not exists for 29-th check, retry later table test.table5 not exists for 2-th check, retry later table multi_rocks.finish_mark not exists for 6-th check, retry later [2024/05/06 15:19:14.570 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331584b00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:13161, start at 2024-05-06 15:19:14.989412146 +0800 CST m=+7.911545329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:15.012 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:15.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:15.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later [2024/05/06 15:19:15.736 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] table mark.finish_mark_3 not exists for 30-th check, retry later [2024/05/06 15:19:16.021 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] table multi_rocks.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 24-th check, retry later table test.table5 exists table test.table6 not exists for 1-th check, retry later [2024/05/06 15:19:16.583 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:16 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00390af30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705433096194 MinTableBarrierTs:449571705695240194 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/683820e2-623a-4075-b4c6-7e2b07b25f92 {"id":"683820e2-623a-4075-b4c6-7e2b07b25f92","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae87c 683820e2-623a-4075-b4c6-7e2b07b25f92 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705433096194,"min-table-barrier-ts":449571705695240194,"admin-job-type":0} /tidb/cdc/default/default/task/position/683820e2-623a-4075-b4c6-7e2b07b25f92/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00390af30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705433096194 MinTableBarrierTs:449571705695240194 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/683820e2-623a-4075-b4c6-7e2b07b25f92 {"id":"683820e2-623a-4075-b4c6-7e2b07b25f92","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae87c 683820e2-623a-4075-b4c6-7e2b07b25f92 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705433096194,"min-table-barrier-ts":449571705695240194,"admin-job-type":0} /tidb/cdc/default/default/task/position/683820e2-623a-4075-b4c6-7e2b07b25f92/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 + grep -q 'failed to get info:' {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00390af30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705433096194 MinTableBarrierTs:449571705695240194 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705695240195, checkpointTs: 449571705695240194, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/683820e2-623a-4075-b4c6-7e2b07b25f92 {"id":"683820e2-623a-4075-b4c6-7e2b07b25f92","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae87c 683820e2-623a-4075-b4c6-7e2b07b25f92 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705433096194,"min-table-barrier-ts":449571705695240194,"admin-job-type":0} /tidb/cdc/default/default/task/position/683820e2-623a-4075-b4c6-7e2b07b25f92/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"e+ grep -q 'etcd info' rror":null,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331584b00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:13161, start at 2024-05-06 15:19:14.989412146 +0800 CST m=+7.911545329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:15.012 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:15.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:15.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3315865c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:13248, start at 2024-05-06 15:19:15.099254542 +0800 CST m=+7.937348423 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:15.137 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:15.145 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:15.145 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 31-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:19:17.752 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2024/05/06 15:19:17.828 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark16000] [ts=449571705740853259] [2024/05/06 15:19:17.828 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark16000] [2024/05/06 15:19:17.828 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571705740853259] [2024/05/06 15:19:17.831 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:17.834 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:17.839 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:17.844 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:17.844 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 15:19:18.035 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] table test.finish_mark not exists for 25-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_rocks.finish_mark not exists for 8-th check, retry later table common_1.finish_mark not exists for 2-th check, retry later table test.table6 exists check diff failed 1-th time, retry later [2024/05/06 15:19:18.596 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later check diff successfully check_ts_not_forward ddl-only-block-related-table table test.finish_mark not exists for 26-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2024/05/06 15:19:20.050 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] table multi_rocks.finish_mark not exists for 9-th check, retry later table common_1.finish_mark not exists for 3-th check, retry later [2024/05/06 15:19:20.618 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2024/05/06 15:19:20.643 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark17000] [ts=449571706684309509] [2024/05/06 15:19:20.643 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark17000] [2024/05/06 15:19:20.643 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571706684309509] [2024/05/06 15:19:20.719 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:20.728 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:20.742 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:20.828 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:20.829 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] table mark.finish_mark_3 not exists for 32-th check, retry later table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later Starting Upstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table test.finish_mark not exists for 27-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] // dir table multi_rocks.finish_mark not exists for 10-th check, retry later [Pipeline] } table common_1.finish_mark exists check diff successfully [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [2024/05/06 15:19:22.069 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14500.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table mark.finish_mark_3 not exists for 33-th check, retry later [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:23 CST 2024] <<<<<< run test case common_1 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Resume changefeed with checkpointTs1 449571702207676422 table test.finish_mark not exists for 28-th check, retry later table test.table3 not exists for 1-th check, retry later [2024/05/06 15:19:24.083 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2024/05/06 15:19:24.098 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark18000] [ts=449571707654504453] [2024/05/06 15:19:24.098 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark18000] [2024/05/06 15:19:24.098 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571707654504453] [2024/05/06 15:19:24.100 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:24.104 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:24.107 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:24.111 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:24.112 +08:00] [INFO] [case.go:571] ["downstream start wait for table"] [tblName=finishmark19000] + set +x + tso='449571707259977729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571707259977729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:19:23 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1453414536.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table multi_rocks.finish_mark not exists for 11-th check, retry later table mark.finish_mark_3 not exists for 34-th check, retry later table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later table test.table3 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 29-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table multi_rocks.finish_mark not exists for 12-th check, retry later [2024/05/06 15:19:26.113 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] table mark.finish_mark_3 not exists for 35-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbe959e0-ed4c-4a94-944c-82952fa00597 {"id":"fbe959e0-ed4c-4a94-944c-82952fa00597","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc52c6af8 fbe959e0-ed4c-4a94-944c-82952fa00597 /tidb/cdc/default/default/upstream/7365782769091768370 {"id":7365782769091768370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbe959e0-ed4c-4a94-944c-82952fa00597 {"id":"fbe959e0-ed4c-4a94-944c-82952fa00597","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc52c6af8 fbe959e0-ed4c-4a94-944c-82952fa00597 /tidb/cdc/default/default/upstream/7365782769091768370 {"id":7365782769091768370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbe959e0-ed4c-4a94-944c-82952fa00597 {"id":"fbe959e0-ed4c-4a94-944c-82952fa00597","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc52c6af8 fbe959e0-ed4c-4a94-944c-82952fa00597 /tidb/cdc/default/default/upstream/7365782769091768370 {"id":7365782769091768370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14586.out cli changefeed create --start-ts=449571707259977729 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later Create changefeed successfully! ID: a982c36e-2689-427d-94e8-603d4ffb6c58 Info: {"upstream_id":7365782769091768370,"namespace":"default","id":"a982c36e-2689-427d-94e8-603d4ffb6c58","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:19:27.369690851+08:00","start_ts":449571707259977729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571707259977729,"checkpoint_ts":449571707259977729,"checkpoint_time":"2024-05-06 15:19:22.387"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.table3 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 30-th check, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table multi_rocks.finish_mark exists check diff successfully [2024/05/06 15:19:28.127 +08:00] [INFO] [case.go:638] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2024/05/06 15:19:28.230 +08:00] [INFO] [case.go:673] ["get end ts"] [tableName=finishmark19000] [ts=449571708637544453] [2024/05/06 15:19:28.230 +08:00] [INFO] [case.go:576] ["ddl synced"] [table=finishmark19000] [2024/05/06 15:19:28.230 +08:00] [INFO] [case.go:582] ["downstream sync success"] [endTs=449571708637544453] [2024/05/06 15:19:28.319 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:28.324 +08:00] [INFO] [case.go:186] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:28.330 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2024/05/06 15:19:28.419 +08:00] [INFO] [case.go:303] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2024/05/06 15:19:28.420 +08:00] [INFO] [case.go:603] ["one downstream exit due to receive done"] [2024/05/06 15:19:28.420 +08:00] [INFO] [case.go:615] ["bank test finished"] [valid=19] [tries=19] [ratio=1] table mark.finish_mark_3 not exists for 36-th check, retry later + cleanup_process cdc.test wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... table savepoint.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:29 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33163f7c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:29462, start at 2024-05-06 15:19:26.948356919 +0800 CST m=+5.500980625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:26.957 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:26.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:26.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33163f7c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:29462, start at 2024-05-06 15:19:26.948356919 +0800 CST m=+5.500980625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:26.957 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:26.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:26.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331640400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:29545, start at 2024-05-06 15:19:27.041283362 +0800 CST m=+5.516563896 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:27.049 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:27.042 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:27.042 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 31-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully wait process cdc.test exit for 3-th time... table test.table3 exists table test.table4 not exists for 1-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 15:19:30 CST 2024] <<<<<< run test case bank success! >>>>>>' [Mon May 6 15:19:30 CST 2024] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster table mark.finish_mark_3 not exists for 37-th check, retry later table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later check_redo_resolved_ts 3d83880a-de7b-4723-9f25-7005399774cf 449571709298671625 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/3d83880a-de7b-4723-9f25-7005399774cf/meta global resolved ts 449571708695740418 not forward to 449571709298671625 run task failed 1-th time, retry later table savepoint.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 32-th check, retry later table test.table4 exists check diff failed 1-th time, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Mon May 6 15:19:31 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3092430926.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:19:28.789 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.92476878s"] check_redo_resolved_ts 3d83880a-de7b-4723-9f25-7005399774cf 449571709298671625 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/3d83880a-de7b-4723-9f25-7005399774cf/meta global resolved ts 449571709220028418 not forward to 449571709298671625 run task failed 2-th time, retry later table mark.finish_mark_3 not exists for 38-th check, retry later table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials table savepoint.finish_mark exists check diff successfully [Pipeline] } table test.finish_mark not exists for 33-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage wait process cdc.test exit for 1-th time... [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } check diff successfully empty select id, val from test.availability3 where id=1 run task successfully delete_owner_key pass wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 39-th check, retry later wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:34 CST 2024] <<<<<< run test case savepoint success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ab0d883-bfa8-444f-8ee0-1955e2367beb {"id":"6ab0d883-bfa8-444f-8ee0-1955e2367beb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc56287f0 6ab0d883-bfa8-444f-8ee0-1955e2367beb /tidb/cdc/default/default/upstream/7365782834800873833 {"id":7365782834800873833,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ab0d883-bfa8-444f-8ee0-1955e2367beb {"id":"6ab0d883-bfa8-444f-8ee0-1955e2367beb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc56287f0 6ab0d883-bfa8-444f-8ee0-1955e2367beb /tidb/cdc/default/default/upstream/7365782834800873833 {"id":7365782834800873833,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6ab0d883-bfa8-444f-8ee0-1955e2367beb {"id":"6ab0d883-bfa8-444f-8ee0-1955e2367beb","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc56287f0 6ab0d883-bfa8-444f-8ee0-1955e2367beb /tidb/cdc/default/default/upstream/7365782834800873833 {"id":7365782834800873833,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 04a25cfc-382c-4f21-b5eb-52c24e568336 Info: {"upstream_id":7365782834800873833,"namespace":"default","id":"04a25cfc-382c-4f21-b5eb-52c24e568336","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:19:35.21828576+08:00","start_ts":449571710590779394,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571710590779394,"checkpoint_ts":449571710590779394,"checkpoint_time":"2024-05-06 15:19:35.093"} table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_kill_capture [Mon May 6 15:19:35 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.51395141.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 34-th check, retry later table autorandom_test.table_a not exists for 1-th check, retry later check_redo_resolved_ts 3d83880a-de7b-4723-9f25-7005399774cf 449571709298671625 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/3d83880a-de7b-4723-9f25-7005399774cf/meta run task successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:19:36 CST 2024] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.75287530.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 40-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/05/06 15:19:38.075 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:19:38.076 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:19:38.076 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571703898505230] [resolvedTs=449571710268604418] [2024/05/06 15:19:38.077 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:19:38.077 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:19:38.078 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.525747ms] [2024/05/06 15:19:38.078 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.538366ms] [2024/05/06 15:19:38.082 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:19:38.084 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:19:38.084 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:19:38.085 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:19:38.086 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:19:38.086 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] table test.finish_mark not exists for 35-th check, retry later [2024/05/06 15:19:38.089 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:19:38.091 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 15:19:38.091 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449571703898505224,\"CommitTs\":449571703898505230,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file_large_value`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file_large_value`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_file_large_value\",\"tbl-name\":\"t1\",\"tbl-id\":108,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 15:19:38.293 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571710268604418] [2024/05/06 15:19:38.293 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571710268604418] table autorandom_test.table_a exists check diff failed 1-th time, retry later [2024/05/06 15:19:38.594 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571710268604418] [2024/05/06 15:19:38.594 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571710268604418] table mark.finish_mark_3 not exists for 41-th check, retry later [2024/05/06 15:19:38.694 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=8] [appliedDDLCount=0] [currentCheckpoint=449571710268604418] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:19:38.694 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] Apply redo log successfully check diff successfully [Mon May 6 15:19:38 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037878c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571710256283658 MinTableBarrierTs:449571711304859654 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571710256283658,"min-table-barrier-ts":449571711567003654,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037878c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571710256283658 MinTableBarrierTs:449571711304859654 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571710256283658,"min-table-barrier-ts":449571711567003654,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037878c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571710256283658 MinTableBarrierTs:449571711304859654 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571710256283658, checkpointTs: 449571710256283658, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571710256283658,"min-table-barrier-ts":449571711567003654,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table processor_delay.t50 exists \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45db3b0b-a6ef-42d8-ac07-00d5099830b7 {"id":"45db3b0b-a6ef-42d8-ac07-00d5099830b7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df883b 45db3b0b-a6ef-42d8-ac07-00d5099830b7 /tidb/cdc/default/default/changefeed/info/54db9b21-45e0-45a5-aa42-419a0324bd76 {"upstream-id":7365782683094104966,"namespace":"default","changefeed-id":"54db9b21-45e0-45a5-aa42-419a0324bd76","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:19:02.965749707+08:00","start-ts":449571702207676422,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571707516092421} /tidb/cdc/default/default/changefeed/status/54db9b21-45e0-45a5-aa42-419a0324bd76 {"checkpoint-ts":449571704121065497,"min-table-barrier-ts":449571704278614043,"admin-job-type":0} /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45db3b0b-a6ef-42d8-ac07-00d5099830b7 {"id":"45db3b0b-a6ef-42d8-ac07-00d5099830b7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df883b 45db3b0b-a6ef-42d8-ac07-00d5099830b7 /tidb/cdc/default/default/changefeed/info/54db9b21-45e0-45a5-aa42-419a0324bd76 {"upstream-id":7365782683094104966,"namespace":"default","changefeed-id":"54db9b21-45e0-45a5-aa42-419a0324bd76","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:19:02.965749707+08:00","start-ts":449571702207676422,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571707516092421} /tidb/cdc/default/default/changefeed/status/54db9b21-45e0-45a5-aa42-419a0324bd76 {"checkpoint-ts":449571704121065497,"min-table-barrier-ts":449571704278614043,"admin-job-type":0} /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/45db3b0b-a6ef-42d8-ac07-00d5099830b7 {"id":"45db3b0b-a6ef-42d8-ac07-00d5099830b7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4df883b 45db3b0b-a6ef-42d8-ac07-00d5099830b7 /tidb/cdc/default/default/changefeed/info/54db9b21-45e0-45a5-aa42-419a0324bd76 {"upstream-id":7365782683094104966,"namespace":"default","changefeed-id":"54db9b21-45e0-45a5-aa42-419a0324bd76","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:19:02.965749707+08:00","start-ts":449571702207676422,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571707516092421} /tidb/cdc/default/default/changefeed/status/54db9b21-45e0-45a5-aa42-419a0324bd76 {"checkpoint-ts":449571704121065497,"min-table-barrier-ts":449571704278614043,"admin-job-type":0} /tidb/cdc/default/default/upstream/7365782683094104966 {"id":7365782683094104966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 54db9b21-45e0-45a5-aa42-419a0324bd76 failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=54db9b21-45e0-45a5-aa42-419a0324bd76 + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 54db9b21-45e0-45a5-aa42-419a0324bd76 -s [Pipeline] // withCredentials table test.finish_mark not exists for 36-th check, retry later [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } "is-owner": true, run task successfully [Pipeline] // withEnv [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } + info='{ "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "failed", "checkpoint_tso": 449571704121065497, "checkpoint_time": "2024-05-06 15:19:10.413", "error": { "time": "2024-05-06T15:19:39.622681256+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "failed", "checkpoint_tso": 449571704121065497, "checkpoint_time": "2024-05-06 15:19:10.413", "error": { "time": "2024-05-06T15:19:39.622681256+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "failed", "checkpoint_tso": 449571704121065497, "checkpoint_time": "2024-05-06 15:19:10.413", "error": { "time": "2024-05-06T15:19:39.622681256+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7365782683094104966, '"namespace":' '"default",' '"id":' '"54db9b21-45e0-45a5-aa42-419a0324bd76",' '"state":' '"failed",' '"checkpoint_tso":' 449571704121065497, '"checkpoint_time":' '"2024-05-06' '15:19:10.413",' '"error":' '{' '"time":' '"2024-05-06T15:19:39.622681256+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365782683094104966, '"namespace":' '"default",' '"id":' '"54db9b21-45e0-45a5-aa42-419a0324bd76",' '"state":' '"failed",' '"checkpoint_tso":' 449571704121065497, '"checkpoint_time":' '"2024-05-06' '15:19:10.413",' '"error":' '{' '"time":' '"2024-05-06T15:19:39.622681256+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully check diff successfully [Pipeline] // podTemplate [Pipeline] } table mark.finish_mark_3 not exists for 42-th check, retry later wait process cdc.test exit for 1-th time... check_changefeed_state http://127.0.0.1:2379 54db9b21-45e0-45a5-aa42-419a0324bd76 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=54db9b21-45e0-45a5-aa42-419a0324bd76 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 54db9b21-45e0-45a5-aa42-419a0324bd76 -s [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + info='{ "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "normal", "checkpoint_tso": 449571712156041220, "checkpoint_time": "2024-05-06 15:19:41.064", "error": null }' + echo '{ "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "normal", "checkpoint_tso": 449571712156041220, "checkpoint_time": "2024-05-06 15:19:41.064", "error": null }' { "upstream_id": 7365782683094104966, "namespace": "default", "id": "54db9b21-45e0-45a5-aa42-419a0324bd76", "state": "normal", "checkpoint_tso": 449571712156041220, "checkpoint_time": "2024-05-06 15:19:41.064", "error": null } ++ echo '{' '"upstream_id":' 7365782683094104966, '"namespace":' '"default",' '"id":' '"54db9b21-45e0-45a5-aa42-419a0324bd76",' '"state":' '"normal",' '"checkpoint_tso":' 449571712156041220, '"checkpoint_time":' '"2024-05-06' '15:19:41.064",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365782683094104966, '"namespace":' '"default",' '"id":' '"54db9b21-45e0-45a5-aa42-419a0324bd76",' '"state":' '"normal",' '"checkpoint_tso":' 449571712156041220, '"checkpoint_time":' '"2024-05-06' '15:19:41.064",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 37-th check, retry later owner pid: 5144 owner id 6d083a91-6106-4622-9a6f-753b95c172f7 table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:42 CST 2024] <<<<<< run test case autorandom success! >>>>>> check diff successfully table mark.finish_mark_3 not exists for 43-th check, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table test1.table1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Mon May 6 15:19:43 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.52745276.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... table test.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:19:44 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> table mark.finish_mark_3 not exists for 44-th check, retry later wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:19:45 CST 2024] <<<<<< run test case tidb_mysql_test success! >>>>>> table test1.table1 not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 45-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:19:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002e0eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571713415118850 MinTableBarrierTs:449571713415118850 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/cb015b76-17a4-411f-b068-dd99bd0af59c {"id":"cb015b76-17a4-411f-b068-dd99bd0af59c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81875 cb015b76-17a4-411f-b068-dd99bd0af59c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571713415118850,"min-table-barrier-ts":449571713415118850,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb015b76-17a4-411f-b068-dd99bd0af59c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002e0eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571713415118850 MinTableBarrierTs:449571713415118850 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/cb015b76-17a4-411f-b068-dd99bd0af59c {"id":"cb015b76-17a4-411f-b068-dd99bd0af59c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81875 cb015b76-17a4-411f-b068-dd99bd0af59c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571713415118850,"min-table-barrier-ts":449571713415118850,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb015b76-17a4-411f-b068-dd99bd0af59c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002e0eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571713415118850 MinTableBarrierTs:449571713415118850 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d083a91-6106-4622-9a6f-753b95c172f7 {"id":"6d083a91-6106-4622-9a6f-753b95c172f7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/cb015b76-17a4-411f-b068-dd99bd0af59c {"id":"cb015b76-17a4-411f-b068-dd99bd0af59c","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81839 6d083a91-6106-4622-9a6f-753b95c172f7 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81875 cb015b76-17a4-411f-b068-dd99bd0af59c /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571713415118850,"min-table-barrier-ts":449571713415118850,"admin-job-type":0} /tidb/cdc/default/default/task/position/6d083a91-6106-4622-9a6f-753b95c172f7/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cb015b76-17a4-411f-b068-dd99bd0af59c/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "6d083a91-6106-4622-9a6f-753b95c172f7" | grep id Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table test1.table1 exists table test1.table2 not exists for 1-th check, retry later "cluster-id": "default" "id": "cb015b76-17a4-411f-b068-dd99bd0af59c", "cluster-id": "default" run task successfully Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 46-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test1.table2 exists check diff successfully nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCD53704153E60 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 07:19:50 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 47-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 48-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:19:53 CST 2024] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 49-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later table mark.finish_mark_3 not exists for 50-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3317e9940012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:6938, start at 2024-05-06 15:19:54.242120787 +0800 CST m=+5.425783800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:54.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:54.213 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:54.213 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3317e9940012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:6938, start at 2024-05-06 15:19:54.242120787 +0800 CST m=+5.425783800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:54.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:54.213 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:54.213 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3317ea740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:7015, start at 2024-05-06 15:19:54.308640804 +0800 CST m=+5.441044236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:54.316 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:54.318 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:54.318 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:19:56.847 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] [Mon May 6 15:19:59 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.82718273.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 51-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331810540006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:15614, start at 2024-05-06 15:19:56.702340907 +0800 CST m=+5.608371527 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:56.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:56.693 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:56.693 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331810540006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:15614, start at 2024-05-06 15:19:56.702340907 +0800 CST m=+5.608371527 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:56.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:56.693 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:56.693 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331811400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:15695, start at 2024-05-06 15:19:56.796032742 +0800 CST m=+5.618981200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:21:56.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:19:56.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:09:56.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 52-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16932.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/912ce993-f85f-410b-bb2b-26e3a053d33b {"id":"912ce993-f85f-410b-bb2b-26e3a053d33b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf67f3 912ce993-f85f-410b-bb2b-26e3a053d33b /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/912ce993-f85f-410b-bb2b-26e3a053d33b {"id":"912ce993-f85f-410b-bb2b-26e3a053d33b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf67f3 912ce993-f85f-410b-bb2b-26e3a053d33b /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/912ce993-f85f-410b-bb2b-26e3a053d33b {"id":"912ce993-f85f-410b-bb2b-26e3a053d33b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf67f3 912ce993-f85f-410b-bb2b-26e3a053d33b /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: e26eb23a-fcbc-421f-ba32-40d6c4d6794b Info: {"upstream_id":7365782951458528569,"namespace":"default","id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:20:02.463357053+08:00","start_ts":449571717726339078,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571717726339078,"checkpoint_ts":449571717726339078,"checkpoint_time":"2024-05-06 15:20:02.313"} Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 53-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_capture [Mon May 6 15:20:03 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.54485450.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } + set +x + tso='449571717590024193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571717590024193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449571717590024193 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 15:20:03 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1696816970.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... [Pipeline] // dir [Pipeline] } table kill_owner_with_ddl.t1 not exists for 1-th check, retry later [Pipeline] // withCredentials [Pipeline] } Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 54-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table kill_owner_with_ddl.t1 exists wait process 8276 exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8a480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571717347278853 MinTableBarrierTs:449571718657998854 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571717347278853,"min-table-barrier-ts":449571718657998854,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8a480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571717347278853 MinTableBarrierTs:449571718657998854 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571717347278853,"min-table-barrier-ts":449571718657998854,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f8a480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571717347278853 MinTableBarrierTs:449571718657998854 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571717347278853, checkpointTs: 449571717347278853, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571717347278853,"min-table-barrier-ts":449571718657998854,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/73657821+ grep -q 'etcd info' 49811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' wait process 8276 exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8743f843-5616-489c-b2c6-6f01832dd004 {"id":"8743f843-5616-489c-b2c6-6f01832dd004","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5e01cf5 8743f843-5616-489c-b2c6-6f01832dd004 /tidb/cdc/default/default/upstream/7365782958932394325 {"id":7365782958932394325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8743f843-5616-489c-b2c6-6f01832dd004 {"id":"8743f843-5616-489c-b2c6-6f01832dd004","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5e01cf5 8743f843-5616-489c-b2c6-6f01832dd004 /tidb/cdc/default/default/upstream/7365782958932394325 {"id":7365782958932394325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8743f843-5616-489c-b2c6-6f01832dd004 {"id":"8743f843-5616-489c-b2c6-6f01832dd004","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5e01cf5 8743f843-5616-489c-b2c6-6f01832dd004 /tidb/cdc/default/default/upstream/7365782958932394325 {"id":7365782958932394325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449571717590024193 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.17014.out cli changefeed create --start-ts=449571717590024193 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml table mark.finish_mark_3 not exists for 55-th check, retry later Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365782958932394325,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:20:06.796079606+08:00","start_ts":449571717590024193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571717590024193,"checkpoint_ts":449571717590024193,"checkpoint_time":"2024-05-06 15:20:01.793"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8276) - No such process wait process 8276 exit for 3-th time... process 8276 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:20:07 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.84188420.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release "is-owner": true, run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. + set +x tidb_server_version 179 Bootstrap version. Do not delete. ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. % Total % Received % Xferd Average Speed Time Time Time Current system_tz Asia/Shanghai TiDB Global System Timezone. Dload Upload Total Spent Left Speed new_collation_enabled True If the new collations are enabled. Do not edit it. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1716 0 --:--:-- --:--:-- --:--:-- 1713 100 221 100 221 0 0 1714 0 --:--:-- --:--:-- --:--:-- 1713 ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331899480004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_uuid 63d331894580018 Current GC worker leader UUID. (DO NOT EDIT) + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 15:20:08.193","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 15:20:08.000","info":"Data syncing is finished"}' tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:32058, start at 2024-05-06 15:20:05.182893829 +0800 CST m=+5.439259241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:15586, start at 2024-05-06 15:20:05.461718851 +0800 CST m=+5.412034155 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:05.191 +0800 Current GC worker leader lease. (DO NOT EDIT) ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '15:20:08.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '15:20:08.000","info":"Data' syncing is 'finished"}' ++ jq .synced tikv_gc_leader_lease 20240506-15:22:05.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_enable true Current GC enable status tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:05.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_safe_point 20240506-15:10:05.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) tikv_gc_last_run_time 20240506-15:20:05.458 +0800 The time when last GC starts. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT tikv_gc_safe_point 20240506-15:10:05.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331899480004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_uuid 63d331894580018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:15586, start at 2024-05-06 15:20:05.461718851 +0800 CST m=+5.412034155 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:05.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:32058, start at 2024-05-06 15:20:05.182893829 +0800 CST m=+5.439259241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_leader_lease 20240506-15:22:05.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:05.458 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_safe_point 20240506-15:10:05.458 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... tikv_gc_last_run_time 20240506-15:20:05.192 +0800 The time when last GC starts. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tikv_gc_safe_point 20240506-15:10:05.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) tidb_server_version 179 Bootstrap version. Do not delete. Verifying Downstream TiDB is started... system_tz Asia/Shanghai TiDB Global System Timezone. VARIABLE_NAME VARIABLE_VALUE COMMENT new_collation_enabled True If the new collations are enabled. Do not edit it. bootstrapped True Bootstrap flag. Do not delete. ddl_table_version 3 DDL Table Version. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. tikv_gc_leader_uuid 63d331899300014 Current GC worker leader UUID. (DO NOT EDIT) system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331895d0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:15666, start at 2024-05-06 15:20:05.486034546 +0800 CST m=+5.361893980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:05.493 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-hgrbq, pid:32121, start at 2024-05-06 15:20:05.247247919 +0800 CST m=+5.421475402 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_leader_lease 20240506-15:22:05.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_enable true Current GC enable status tikv_gc_last_run_time 20240506-15:20:05.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_safe_point 20240506-15:10:05.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. Starting Upstream TiFlash... tikv_gc_last_run_time 20240506-15:20:05.236 +0800 The time when last GC starts. (DO NOT EDIT) TiFlash tikv_gc_safe_point 20240506-15:10:05.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Release Version: v7.5.1-12-g9002cc34d Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD Edition: Community UTC Build Time: 2024-04-18 07:24:48 Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO UTC Build Time: 2024-04-18 07:24:48 Raft Proxy Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD Raft Proxy UTC Build Time: 2024-04-18 07:28:40 Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Git Commit Branch: HEAD Prometheus Prefix: tiflash_proxy_ Profile: release UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Prometheus Prefix: tiflash_proxy_ Profile: release Verifying Upstream TiFlash is started... Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Verifying Upstream TiFlash is started... Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:20:07.538 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=2892346218] ["get source checksum cost"=2.395259ms] ["get target checksum cost"=1.349459ms] + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '15:20:08.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '15:20:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 15:20:08.193' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '15:20:08.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '15:20:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '15:20:08.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '15:20:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 15:20:08' + echo 'sink_checkpoint_ts is 2024-05-06' 15:20:08.193 sink_checkpoint_ts is 2024-05-06 15:20:08.193 ++ date -d '2024-05-06 15:20:08.193' +%s + checkpoint_timestamp=1714980008 ++ date -d '2024-05-06 15:20:08' +%s + current_timestamp=1714980008 + '[' 0 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later owner pid: 5453 owner id 466211ab-b11e-4dc2-b27a-50bd4f7828d1 [Mon May 6 15:20:09 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.55545556.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 56-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002da1710 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719705788425 MinTableBarrierTs:449571719705788425 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719705788425, checkpointTs: 449571719705788425, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb53674a-10d6-4616-ad4d-a833333408da {"id":"eb53674a-10d6-4616-ad4d-a833333408da","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf685a eb53674a-10d6-4616-ad4d-a833333408da /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719705788425,"min-table-barrier-ts":449571719705788425,"admin-job-type":0} /tidb/cdc/default/default/task/position/eb53674a-10d6-4616-ad4d-a833333408da/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002da1710 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719705788425 MinTableBarrierTs:449571719705788425 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719705788425, checkpointTs: 449571719705788425, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb53674a-10d6-4616-ad4d-a833333408da {"id":"eb53674a-10d6-4616-ad4d-a833333408da","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf685a eb53674a-10d6-4616-ad4d-a833333408da /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719705788425,"min-table-barrier-ts":449571719705788425,"admin-job-type":0} /tidb/cdc/default/default/task/position/eb53674a-10d6-4616-ad4d-a833333408da/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002da1710 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719705788425 MinTableBarrierTs:449571719705788425 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719705788425, checkpointTs: 449571719705788425, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb53674a-10d6-4616-ad4d-a833333408da {"id":"eb53674a-10d6-4616-ad4d-a833333408da","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf685a eb53674a-10d6-4616-ad4d-a833333408da /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719705788425,"min-table-barrier-ts":449571719705788425,"admin-job-type":0} /tidb/cdc/default/default/task/position/eb53674a-10d6-4616-ad4d-a833333408da/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.33351.out cli tso query --pd=http://127.0.0.1:2379 table test.t1 exists + sleep 5 [Mon May 6 15:20:10 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1699516997.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 57-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449571719857307649 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571719857307649 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:20:11 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3339033392.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309a6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571719968718859 MinTableBarrierTs:449571720230862858 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/9f41a32d-8887-4706-91ff-9f951dcd3681 {"id":"9f41a32d-8887-4706-91ff-9f951dcd3681","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81909 9f41a32d-8887-4706-91ff-9f951dcd3681 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571719968718859,"min-table-barrier-ts":449571720230862858,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9f41a32d-8887-4706-91ff-9f951dcd3681/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309a6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571719968718859 MinTableBarrierTs:449571720230862858 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/9f41a32d-8887-4706-91ff-9f951dcd3681 {"id":"9f41a32d-8887-4706-91ff-9f951dcd3681","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81909 9f41a32d-8887-4706-91ff-9f951dcd3681 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571719968718859,"min-table-barrier-ts":449571720230862858,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9f41a32d-8887-4706-91ff-9f951dcd3681/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert+ grep -q 'failed to get info:' -allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309a6c0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571719968718859 MinTableBarrierTs:449571720230862858 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/466211ab-b11e-4dc2-b27a-50bd4f7828d1 {"id":"466211ab-b11e-4dc2-b27a-50bd4f7828d1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/9f41a32d-8887-4706-91ff-9f951dcd3681 {"id":"9f41a32d-8887-4706-91ff-9f951dcd3681","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f818db 466211ab-b11e-4dc2-b27a-50bd4f7828d1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f81909 9f41a32d-8887-4706-91ff-9f951dcd3681 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571719968718859,"min-table-barrier-ts":449571720230862858,"admin-job-type":0} /tidb/cdc/default/default/task/position/466211ab-b11e-4dc2-b27a-50bd4f7828d1/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/9f41a32d-8887-4706-91ff-9f951dcd3681/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "466211ab-b11e-4dc2-b27a-50bd4f7828d1" | grep id table mark.finish_mark_3 not exists for 58-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331902300006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:6897, start at 2024-05-06 15:20:12.178247454 +0800 CST m=+5.015093859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:12.184 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:12.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:12.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331902300006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:6897, start at 2024-05-06 15:20:12.178247454 +0800 CST m=+5.015093859 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:12.184 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:12.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:12.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331905380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:6974, start at 2024-05-06 15:20:12.391948267 +0800 CST m=+5.166245848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:12.398 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:12.366 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:12.366 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } "id": "9f41a32d-8887-4706-91ff-9f951dcd3681", "cluster-id": "default" "cluster-id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:13 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46fc0dbe-97e3-4ffc-aa98-a9262f79116e {"id":"46fc0dbe-97e3-4ffc-aa98-a9262f79116e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc60321f1 46fc0dbe-97e3-4ffc-aa98-a9262f79116e /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46fc0dbe-97e3-4ffc-aa98-a9262f79116e {"id":"46fc0dbe-97e3-4ffc-aa98-a9262f79116e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc60321f1 46fc0dbe-97e3-4ffc-aa98-a9262f79116e /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/46fc0dbe-97e3-4ffc-aa98-a9262f79116e {"id":"46fc0dbe-97e3-4ffc-aa98-a9262f79116e","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc60321f1 46fc0dbe-97e3-4ffc-aa98-a9262f79116e /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table mark.finish_mark_3 not exists for 59-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:15 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f33b26-e438-41f9-a1c4-fada2d5f9dd1 {"id":"26f33b26-e438-41f9-a1c4-fada2d5f9dd1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5fc0cf6 26f33b26-e438-41f9-a1c4-fada2d5f9dd1 /tidb/cdc/default/default/upstream/7365782998948959195 {"id":7365782998948959195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f33b26-e438-41f9-a1c4-fada2d5f9dd1 {"id":"26f33b26-e438-41f9-a1c4-fada2d5f9dd1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5fc0cf6 26f33b26-e438-41f9-a1c4-fada2d5f9dd1 /tidb/cdc/default/default/upstream/7365782998948959195 {"id":7365782998948959195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f33b26-e438-41f9-a1c4-fada2d5f9dd1 {"id":"26f33b26-e438-41f9-a1c4-fada2d5f9dd1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5fc0cf6 26f33b26-e438-41f9-a1c4-fada2d5f9dd1 /tidb/cdc/default/default/upstream/7365782998948959195 {"id":7365782998948959195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.33434.out cli changefeed create --start-ts=449571719857307649 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="5000" "insertproportion"="0" "mysql.user"="root" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" "scanproportion"="0" "mysql.db"="consistent_replicate_storage_s3" "fieldcount"="100" "operationcount"="0" "requestdistribution"="uniform" "workload"="core" "readproportion"="0" "threadcount"="10" ********************************************** Create changefeed successfully! ID: bb04560b-f9d0-429b-af2e-9f92a20ced9d Info: {"upstream_id":7365782998948959195,"namespace":"default","id":"bb04560b-f9d0-429b-af2e-9f92a20ced9d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:20:15.402887637+08:00","start_ts":449571719857307649,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571719857307649,"checkpoint_ts":449571719857307649,"checkpoint_time":"2024-05-06 15:20:10.442"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... skip some DDLs in tidb v4.0.x [Mon May 6 15:20:15 CST 2024] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.82798281.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33192c34000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:7439, start at 2024-05-06 15:20:14.873222413 +0800 CST m=+5.171335694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:14.880 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:14.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:14.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33192c34000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:7439, start at 2024-05-06 15:20:14.873222413 +0800 CST m=+5.171335694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:14.880 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:14.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:14.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33192ce80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-gwd51, pid:7520, start at 2024-05-06 15:20:14.945115863 +0800 CST m=+5.192301266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:14.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:14.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:14.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2429 0 --:--:-- --:--:-- --:--:-- 2454 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 15:20:15.193","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-06 15:20:08.993","now_ts":"2024-05-06 15:20:16.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '15:20:15.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-06' '15:20:08.993","now_ts":"2024-05-06' '15:20:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '15:20:15.193","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-06' '15:20:08.993","now_ts":"2024-05-06' '15:20:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + set +x table mark.finish_mark_3 not exists for 60-th check, retry later Run finished, takes 2.04381772s INSERT - Takes(s): 2.0, Count: 5000, OPS: 2457.5, Avg(us): 3720, Min(us): 1607, Max(us): 69044, 95th(us): 5000, 99th(us): 36000 nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [Mon May 6 15:20:18 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.88308832.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 8423 exit for 1-th time... wait process 8423 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8423) - No such process wait process 8423 exit for 3-th time... process 8423 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:20:18 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.86078609.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f3cd82-66cd-44b6-9854-70e1f759a47b {"id":"e8f3cd82-66cd-44b6-9854-70e1f759a47b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61b06ee e8f3cd82-66cd-44b6-9854-70e1f759a47b /tidb/cdc/default/default/upstream/7365783032959847152 {"id":7365783032959847152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f3cd82-66cd-44b6-9854-70e1f759a47b {"id":"e8f3cd82-66cd-44b6-9854-70e1f759a47b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61b06ee e8f3cd82-66cd-44b6-9854-70e1f759a47b /tidb/cdc/default/default/upstream/7365783032959847152 {"id":7365783032959847152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8f3cd82-66cd-44b6-9854-70e1f759a47b {"id":"e8f3cd82-66cd-44b6-9854-70e1f759a47b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61b06ee e8f3cd82-66cd-44b6-9854-70e1f759a47b /tidb/cdc/default/default/upstream/7365783032959847152 {"id":7365783032959847152,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a table mark.finish_mark_3 not exists for 61-th check, retry later table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later table generate_column.t1 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > changefeed is working normally rts: 449571722015014900->449571722421076000 checkpoint: 449571722015014900->449571722421076000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 15:20:20.440 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE DATABASE `ddl_reentrant`"] [duration=79.369646ms] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] run task successfully table mark.finish_mark_3 not exists for 62-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d963f95b-c3c5-4337-b030-3eb3cdc9a146 {"id":"d963f95b-c3c5-4337-b030-3eb3cdc9a146","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61f14f0 d963f95b-c3c5-4337-b030-3eb3cdc9a146 /tidb/cdc/default/default/upstream/7365783040887240702 {"id":7365783040887240702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d963f95b-c3c5-4337-b030-3eb3cdc9a146 {"id":"d963f95b-c3c5-4337-b030-3eb3cdc9a146","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61f14f0 d963f95b-c3c5-4337-b030-3eb3cdc9a146 /tidb/cdc/default/default/upstream/7365783040887240702 {"id":7365783040887240702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d963f95b-c3c5-4337-b030-3eb3cdc9a146 {"id":"d963f95b-c3c5-4337-b030-3eb3cdc9a146","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc61f14f0 d963f95b-c3c5-4337-b030-3eb3cdc9a146 /tidb/cdc/default/default/upstream/7365783040887240702 {"id":7365783040887240702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later resume changefeed da178b89-a18b-47ef-872f-ca9f08d58d6a from 449571722054074397 check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later check_changefeed_status 127.0.0.1:8300 fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 836 0 --:--:-- --:--:-- --:--:-- 843 + info='{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' + echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' {"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732} ++ echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/fffc9d8f-d6e0-4fe6-b9b7-efc855eea2b1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:21 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003620480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 {"id":"26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68b4 26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003620480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 {"id":"26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68b4 26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003620480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 {"id":"26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68b4 26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/26fdf5f7-7afe-4fc5-8ac3-ec0e46d81789/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 671 0 --:--:-- --:--:-- --:--:-- 671 + info='{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' + echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' {"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732} ++ echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449571722693705732,"checkpoint_ts":449571722693705732}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later table mark.finish_mark_3 exists table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... check diff successfully table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:20:24 CST 2024] <<<<<< run test case generate_column success! >>>>>> wait process cdc.test exit for 2-th time... table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:20:24 CST 2024] <<<<<< run test case default_value success! >>>>>> table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later changefeed is working normally rts: 449571722421076000->449571722421076000 checkpoint: 449571722054074400->449571723469652000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 15:20:23.359 +08:00] [INFO] [mysql_ddl_sink.go:119] ["Execute DDL failed, but error can be ignored"] [startTs=449571722054074393] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] [error="Error 1007 (HY000): Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a table sink_hang.t2 exists check diff failed 1-th time, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 7847 exit for 1-th time... wait process 7847 exit for 2-th time... wait process 7847 exit for 3-th time... wait process 7847 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7847) - No such process wait process 7847 exit for 5-th time... process 7847 already exit [Mon May 6 15:20:20 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.84178419.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003750f30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705813205006 MinTableBarrierTs:449571723271733255 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b118bbac-3a83-4e3a-9211-2dfdbbfab913 {"id":"b118bbac-3a83-4e3a-9211-2dfdbbfab913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae968 b118bbac-3a83-4e3a-9211-2dfdbbfab913 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705813205006,"min-table-barrier-ts":449571723271733255,"admin-job-type":0} /tidb/cdc/default/default/task/position/b118bbac-3a83-4e3a-9211-2dfdbbfab913/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003750f30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705813205006 MinTableBarrierTs:449571723271733255 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b118bbac-3a83-4e3a-9211-2dfdbbfab913 {"id":"b118bbac-3a83-4e3a-9211-2dfdbbfab913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae968 b118bbac-3a83-4e3a-9211-2dfdbbfab913 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705813205006,"min-table-barrier-ts":449571723271733255,"admin-job-type":0} /tidb/cdc/default/default/task/position/b118bbac-3a83-4e3a-9211-2dfdbbfab913/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":nu+ grep -q 'failed to get info:' ll,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365782699936505141 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 15:19:02.160997837 +0800 CST StartTs:449571701933473793 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003750f30 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571701946581001} {CheckpointTs:449571705813205006 MinTableBarrierTs:449571723271733255 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571705813205006, checkpointTs: 449571705813205006, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b118bbac-3a83-4e3a-9211-2dfdbbfab913 {"id":"b118bbac-3a83-4e3a-9211-2dfdbbfab913","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc4eae968 b118bbac-3a83-4e3a-9211-2dfdbbfab913 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365782699936505141,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T15:19:02.160997837+08:00","start-ts":449571701933473793,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571701946581001} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449571705813205006,"min-table-barrier-ts":449571723271733255,"admin-job-type":0} /tidb/cdc/default/default/task/position/b118bbac-3a83-4e3a-9211-2dfdbbfab913/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782699936505141 {"id":7365782699936505141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449571705813205000->449571723534139400 checkpoint: 449571705813205000->449571723534139400 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:20:27 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> run task failed 1-th time, retry later table big_txn.finish_mark_1 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:20:17 CST 2024] <<<<<< run test case big_txn success! >>>>>> table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 4-th time, retry later check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later wait process 8612 exit for 1-th time... wait process 8612 exit for 2-th time... check diff failed 3-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8612) - No such process wait process 8612 exit for 3-th time... process 8612 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:20:31 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.87558757.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed is working normally rts: 449571724531597300->449571725055885300 checkpoint: 449571724793741300->449571725055885300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 15:20:25.393 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [duration=80.95079ms] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] run task successfully table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later resume changefeed da178b89-a18b-47ef-872f-ca9f08d58d6a from 449571723705581603 check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 4-th time, retry later run task failed 1-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:34 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbc750 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e4f33c8-2fdc-42e9-96b9-bf416e74185b {"id":"0e4f33c8-2fdc-42e9-96b9-bf416e74185b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68f9 0e4f33c8-2fdc-42e9-96b9-bf416e74185b /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e4f33c8-2fdc-42e9-96b9-bf416e74185b/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbc750 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e4f33c8-2fdc-42e9-96b9-bf416e74185b {"id":"0e4f33c8-2fdc-42e9-96b9-bf416e74185b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68f9 0e4f33c8-2fdc-42e9-96b9-bf416e74185b /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e4f33c8-2fdc-42e9-96b9-bf416e74185b/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 + grep -q 'failed to get info:' {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002dbc750 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e4f33c8-2fdc-42e9-96b9-bf416e74185b {"id":"0e4f33c8-2fdc-42e9-96b9-bf416e74185b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf68f9 0e4f33c8-2fdc-42e9-96b9-bf416e74185b /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/0e4f33c8-2fdc-42e9-96b9-bf416e74185b/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } check diff failed 5-th time, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/06 15:20:35.207 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/06 15:20:35.439 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:20:35.454 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:20:35.623 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:20:35.624 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:20:35.655 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:20:35.657 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:20:35.800 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_expire_capture [Mon May 6 15:20:36 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.57735775.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:20:36.878 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] pass test: get status pass test: health pass test: changefeed apis [2024/05/06 15:20:36.020 +08:00] [INFO] [cases.go:413] ["check changefeed failed"] [time=0] [info=null] check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= [2024/05/06 15:20:37.014 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] The 1 times to try to start tidb cluster... [2024/05/06 15:20:37.015 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] The 1 times to try to start tidb cluster... [2024/05/06 15:20:37.088 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:20:37.090 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:20:37.256 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] wait process cdc.test exit for 1-th time... [2024/05/06 15:20:37.331 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 15:20:37.438 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:20:37.439 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] wait process cdc.test exit for 2-th time... table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:20:38 CST 2024] <<<<<< run test case sink_hang success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed is working normally rts: 449571726366343200->449571726890893300 checkpoint: 449571726366343200->449571726890893300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 15:20:32.369 +08:00] [INFO] [mysql_ddl_sink.go:119] ["Execute DDL failed, but error can be ignored"] [startTs=449571723705581594] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] [error="Error 1050 (42S01): Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a [2024/05/06 15:20:38.538 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:20:38.541 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 15:20:38.652 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:20:38.711 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 15:20:38.834 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:20:38.835 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 15:20:38.890 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] pass test: delete changefeed apis pass test: capture apis pass test: processor apis pass test: owner apis pass test: set log level [2024/05/06 15:20:38.893 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 15:20:39.026 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:20:39.078 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] + cleanup_process cdc.test start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1b950 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571727321858050 MinTableBarrierTs:449571727321858050 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571727321858050, checkpointTs: 449571727321858050, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571727321858051, checkpointTs: 449571727321858050, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571727321858053, checkpointTs: 449571727321858050, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 {"id":"0d49bb23-1869-419a-99d9-6e0fd8ecb3e6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8198b 0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571727321858050,"min-table-barrier-ts":449571727321858050,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1b950 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571727321858050 MinTableBarrierTs:449571727321858050 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571727321858050, checkpointTs: 449571727321858050, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571727321858051, checkpointTs: 449571727321858050, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571727321858053, checkpointTs: 449571727321858050, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 {"id":"0d49bb23-1869-419a-99d9-6e0fd8ecb3e6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8198b 0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571727321858050,"min-table-barrier-ts":449571727321858050,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e1b950 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571727321858050 MinTableBarrierTs:449571727321858050 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571727321858050, checkpointTs: 449571727321858050, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571727321858051, checkpointTs: 449571727321858050, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571727321858053, checkpointTs: 449571727321858050, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 {"id":"0d49bb23-1869-419a-99d9-6e0fd8ecb3e6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8198b 0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571727321858050,"min-table-barrier-ts":449571727321858050,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d49bb23-1869-419a-99d9-6e0fd8ecb3e6/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... changefeed is working normally rts: 449571727152775200->449571727152775200 checkpoint: 449571727152775200->449571727205204030 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 15:20:40.593 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [duration=81.362805ms] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] run task successfully "is-owner": true, run task successfully wait process cdc.test exit for 3-th time... resume changefeed da178b89-a18b-47ef-872f-ca9f08d58d6a from 449571727205204001 check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Mon May 6 15:20:41 CST 2024] <<<<<< run test case api_v2 success! >>>>>>' [Mon May 6 15:20:41 CST 2024] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release owner pid: 5778 owner id 0d49bb23-1869-419a-99d9-6e0fd8ecb3e6 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '0d49bb23-1869-419a-99d9-6e0fd8ecb3e6' /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f8198b run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later run task failed 1-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 13-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 8760 exit for 1-th time... wait process 8760 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8760) - No such process wait process 8760 exit for 3-th time... process 8760 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:20:43 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.89038905.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331aec44001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:3912, start at 2024-05-06 15:20:43.585879492 +0800 CST m=+5.192199168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:43.593 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:43.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:43.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331aec44001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:3912, start at 2024-05-06 15:20:43.585879492 +0800 CST m=+5.192199168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:43.593 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:43.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:43.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331aedc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:3999, start at 2024-05-06 15:20:43.654405821 +0800 CST m=+5.205215259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:43.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:43.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:43.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Mon May 6 15:20:46 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.59625964.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_s3.check1 not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:20:46 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.53145316.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00388c1b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ad057f-263e-4c24-a928-17acdbb4e537 {"id":"40ad057f-263e-4c24-a928-17acdbb4e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6936 40ad057f-263e-4c24-a928-17acdbb4e537 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/40ad057f-263e-4c24-a928-17acdbb4e537/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00388c1b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ad057f-263e-4c24-a928-17acdbb4e537 {"id":"40ad057f-263e-4c24-a928-17acdbb4e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6936 40ad057f-263e-4c24-a928-17acdbb4e537 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/40ad057f-263e-4c24-a928-17acdbb4e537/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00388c1b0 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719862812682 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40ad057f-263e-4c24-a928-17acdbb4e537 {"id":"40ad057f-263e-4c24-a928-17acdbb4e537","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6936 40ad057f-263e-4c24-a928-17acdbb4e537 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719862812682,"admin-job-type":0} /tidb/cdc/default/default/task/position/40ad057f-263e-4c24-a928-17acdbb4e537/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 [2024/05/06 15:20:45.770 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/06 15:20:45.985 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:20:45.992 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:20:46.169 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:20:46.170 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:20:46.177 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:20:46.178 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:20:46.355 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 15:20:46.371 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 15:20:46.522 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 15:20:46.523 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 15:20:46.550 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:20:46.553 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:20:46.693 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 15:20:46.735 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 15:20:46.866 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:20:46.867 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 15:20:46.915 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:20:46.918 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 15:20:47.051 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:20:47.099 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 15:20:47.221 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:20:47.222 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] run task successfully [2024/05/06 15:20:47.271 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 15:20:47.274 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 15:20:47.389 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:20:47.451 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 15:20:47.565 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 15:20:47.568 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 15:20:47.625 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 15:20:47.631 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 15:20:47.730 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] changefeed is working normally rts: 449571728988045300->449571729250189300 checkpoint: 449571728988045300->449571729250189300 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2024/05/06 15:20:41.419 +08:00] [INFO] [mysql_ddl_sink.go:119] ["Execute DDL failed, but error can be ignored"] [startTs=449571727205203996] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] [error="Error 1050 (42S01): Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a [2024/05/06 15:20:47.791 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 15:20:47.922 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 15:20:47.925 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 15:20:47.974 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 15:20:47.980 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] table consistent_replicate_storage_s3.check1 not exists for 15-th check, retry later [2024/05/06 15:20:48.091 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 15:20:48.140 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 15:20:48.263 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 15:20:48.266 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [2024/05/06 15:20:48.307 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 15:20:48.313 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 15:20:48.428 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 15:20:48.472 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:20:48.597 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 15:20:48.600 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 15:20:48.643 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 15:20:48.649 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 15:20:48.775 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [Pipeline] // withCredentials ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [2024/05/06 15:20:48.819 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 15:20:48.950 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 15:20:48.953 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 15:20:49.001 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 15:20:49.010 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2024/05/06 15:20:49.131 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 15:20:49.179 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // node + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fca480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571728632578051 MinTableBarrierTs:449571729943298051 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0022426a-803b-4186-bed6-eeacffeb8542 {"id":"0022426a-803b-4186-bed6-eeacffeb8542","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f819d0 0022426a-803b-4186-bed6-eeacffeb8542 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571728632578051,"min-table-barrier-ts":449571729943298051,"admin-job-type":0} /tidb/cdc/default/default/task/position/0022426a-803b-4186-bed6-eeacffeb8542/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782149811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fca480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571728632578051 MinTableBarrierTs:449571729943298051 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0022426a-803b-4186-bed6-eeacffeb8542 {"id":"0022426a-803b-4186-bed6-eeacffeb8542","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f819d0 0022426a-803b-4186-bed6-eeacffeb8542 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571728632578051,"min-table-barrier-ts":449571729943298051,"admin-job-type":0} /tidb/cdc/default/default/task/position/0022426a-803b-4186-bed6-eeacffeb8542/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/73657821+ grep -q 'failed to get info:' 49811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c6a72c52-89de-4b97-94c0-1fc17e986645 {UpstreamID:7365782149811922033 Namespace:default ID:c6a72c52-89de-4b97-94c0-1fc17e986645 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:16:55.466049261 +0800 CST StartTs:449571667369525249 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fca480 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571668732674054} {CheckpointTs:449571728632578051 MinTableBarrierTs:449571729943298051 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571728632578051, checkpointTs: 449571728632578051, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0022426a-803b-4186-bed6-eeacffeb8542 {"id":"0022426a-803b-4186-bed6-eeacffeb8542","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc2f819d0 0022426a-803b-4186-bed6-eeacffeb8542 /tidb/cdc/default/default/changefeed/info/c6a72c52-89de-4b97-94c0-1fc17e986645 {"upstream-id":7365782149811922033,"namespace":"default","changefeed-id":"c6a72c52-89de-4b97-94c0-1fc17e986645","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:16:55.466049261+08:00","start-ts":449571667369525249,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571668732674054} /tidb/cdc/default/default/changefeed/status/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":449571728632578051,"min-table-barrier-ts":449571729943298051,"admin-job-type":0} /tidb/cdc/default/default/task/position/0022426a-803b-4186-bed6-eeacffeb8542/c6a72c52-89de-4b97-94c0-1fc17e986645 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/73657821+ grep -q 'etcd info' 49811922033 {"id":7365782149811922033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Pipeline] } [2024/05/06 15:20:49.309 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 15:20:49.312 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 15:20:49.363 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 15:20:49.376 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 15:20:49.487 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 15:20:49.536 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [Pipeline] // podTemplate [Pipeline] } changefeed is working normally rts: 449571729512333300->449571729774215200 checkpoint: 449571729512333300->449571729774215200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 15:20:48.712 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=149.438079ms] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] run task successfully [Pipeline] // withEnv [Pipeline] } [2024/05/06 15:20:49.665 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 15:20:49.671 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 15:20:49.706 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 15:20:49.718 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [Pipeline] // stage [Pipeline] } table consistent_replicate_storage_s3.check1 not exists for 16-th check, retry later [2024/05/06 15:20:49.833 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 15:20:49.869 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 15:20:49.999 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 15:20:50.008 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 15:20:50.039 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 15:20:50.051 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:49 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e85ceec-3db7-4794-9919-dbc98bf68bb7 {"id":"8e85ceec-3db7-4794-9919-dbc98bf68bb7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc68f1af0 8e85ceec-3db7-4794-9919-dbc98bf68bb7 /tidb/cdc/default/default/upstream/7365783164781034075 {"id":7365783164781034075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e85ceec-3db7-4794-9919-dbc98bf68bb7 {"id":"8e85ceec-3db7-4794-9919-dbc98bf68bb7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc68f1af0 8e85ceec-3db7-4794-9919-dbc98bf68bb7 /tidb/cdc/default/default/upstream/7365783164781034075 {"id":7365783164781034075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8e85ceec-3db7-4794-9919-dbc98bf68bb7 {"id":"8e85ceec-3db7-4794-9919-dbc98bf68bb7","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc68f1af0 8e85ceec-3db7-4794-9919-dbc98bf68bb7 /tidb/cdc/default/default/upstream/7365783164781034075 {"id":7365783164781034075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x resume changefeed da178b89-a18b-47ef-872f-ca9f08d58d6a from 449571729551654917 check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a [2024/05/06 15:20:50.165 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 15:20:50.204 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 15:20:50.329 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 15:20:50.338 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 15:20:50.372 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 15:20:50.386 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 15:20:50.493 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 15:20:50.538 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b4d640009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:21257, start at 2024-05-06 15:20:49.760631133 +0800 CST m=+5.214246462 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.753 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.753 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b46f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:9443, start at 2024-05-06 15:20:49.378555161 +0800 CST m=+5.148491331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.391 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.391 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b46f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:9443, start at 2024-05-06 15:20:49.378555161 +0800 CST m=+5.148491331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.391 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.391 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b479c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-j5hvh, pid:9523, start at 2024-05-06 15:20:49.406654005 +0800 CST m=+5.118360519 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.383 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.383 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:20:50.658 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 15:20:50.670 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 15:20:50.706 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 15:20:50.723 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] "is-owner": true, run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:20:50 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> owner pid: 5967 owner id 0022426a-803b-4186-bed6-eeacffeb8542 table consistent_replicate_storage_s3.check1 not exists for 17-th check, retry later run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b4d640009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:21257, start at 2024-05-06 15:20:49.760631133 +0800 CST m=+5.214246462 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.753 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.753 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331b4e240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:21338, start at 2024-05-06 15:20:49.840820572 +0800 CST m=+5.225355011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:22:49.850 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:20:49.851 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:10:49.851 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later ***************** properties ***************** "insertproportion"="0" "workload"="core" "mysql.port"="4000" "requestdistribution"="uniform" "readproportion"="0" "mysql.db"="changefeed_auto_stop_1" "recordcount"="20" "mysql.host"="127.0.0.1" "readallfields"="true" "updateproportion"="0" "operationcount"="0" "mysql.user"="root" "scanproportion"="0" "dotransactions"="false" "threadcount"="4" ********************************************** Run finished, takes 9.426263ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3833.9, Avg(us): 1771, Min(us): 949, Max(us): 4247, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_2" "dotransactions"="false" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "threadcount"="4" "recordcount"="20" "updateproportion"="0" "mysql.port"="4000" "operationcount"="0" "readproportion"="0" "workload"="core" "readallfields"="true" "scanproportion"="0" ********************************************** Run finished, takes 9.761338ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3565.2, Avg(us): 1813, Min(us): 952, Max(us): 4120, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "updateproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "readproportion"="0" "insertproportion"="0" "operationcount"="0" "threadcount"="4" "mysql.user"="root" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.db"="changefeed_auto_stop_3" "workload"="core" "recordcount"="20" "scanproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 9.817769ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3533.1, Avg(us): 1827, Min(us): 1092, Max(us): 4086, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "requestdistribution"="uniform" "operationcount"="0" "mysql.user"="root" "readproportion"="0" "readallfields"="true" "insertproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="changefeed_auto_stop_4" "scanproportion"="0" "recordcount"="20" "mysql.port"="4000" "threadcount"="4" "updateproportion"="0" ********************************************** Run finished, takes 10.305633ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3568.0, Avg(us): 1923, Min(us): 955, Max(us): 4637, 95th(us): 5000, 99th(us): 5000 [Mon May 6 15:20:53 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1100411006.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 changefeed id: c6a72c52-89de-4b97-94c0-1fc17e986645 curl -s -d "cf-id=c6a72c52-89de-4b97-94c0-1fc17e986645&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 496 765 --:--:-- --:--:-- --:--:-- 771 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:20:53 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> table consistent_replicate_storage_s3.check1 not exists for 18-th check, retry later check_ts_forward da178b89-a18b-47ef-872f-ca9f08d58d6a check diff successfully + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22601.out cli tso query --pd=http://127.0.0.1:2379 nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later table consistent_replicate_storage_s3.check1 not exists for 19-th check, retry later changefeed is working normally rts: 449571731085197300->449571731609223200 checkpoint: 449571731347341300->449571731609223200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2024/05/06 15:20:51.337 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4`"] [duration=920.977609ms] [namespace=default] [changefeed=da178b89-a18b-47ef-872f-ca9f08d58d6a] run task successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:56 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:20:56 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1105411056.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449571731578028036 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571731578028036 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:20:56 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2265122653.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 8908 exit for 1-th time... wait process 8908 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8908) - No such process wait process 8908 exit for 3-th time... process 8908 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 15:20:56 CST 2024] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.90459047.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2024/05/06 15:20:56.675 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/06 15:20:56.863 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:20:56.939 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:20:57.022 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:20:57.023 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:20:57.108 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:20:57.109 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:20:57.195 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:20:57 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> [2024/05/06 15:20:57.592 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:20:57 CST 2024] <<<<<< run test case ddl_reentrant success! >>>>>> [2024/05/06 15:20:57.702 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 15:20:57.703 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 15:20:57.783 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:20:57.784 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:20:57.873 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later [2024/05/06 15:20:57.956 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 15:20:58.038 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:20:58.039 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 15:20:58.139 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 15:20:58.140 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:20:58.221 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:20:58.310 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 15:20:58.384 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:20:58.385 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 15:20:58.472 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 15:20:58.473 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 15:20:58.543 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:20:58.640 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 15:20:58.705 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 15:20:58.706 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 15:20:58.809 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 15:20:58.809 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 15:20:58.870 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 15:20:58.973 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 15:20:59.031 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 15:20:59.032 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 15:20:59.137 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 15:20:59.138 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 15:20:59.198 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 15:20:59.322 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 15:20:59.366 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 15:20:59.366 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 15:20:59.494 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 15:20:59.495 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 15:20:59.529 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 15:20:59.659 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/06 15:20:59.687 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 15:20:59.688 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 15:20:59.829 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 15:20:59.830 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 15:20:59.849 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f08510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719993884708 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770fe66c-57af-4513-80b7-ee5a52ddabb9 {"id":"770fe66c-57af-4513-80b7-ee5a52ddabb9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6984 770fe66c-57af-4513-80b7-ee5a52ddabb9 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719993884708,"admin-job-type":0} /tidb/cdc/default/default/task/position/770fe66c-57af-4513-80b7-ee5a52ddabb9/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f08510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719993884708 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770fe66c-57af-4513-80b7-ee5a52ddabb9 {"id":"770fe66c-57af-4513-80b7-ee5a52ddabb9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6984 770fe66c-57af-4513-80b7-ee5a52ddabb9 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719993884708,"admin-job-type":0} /tidb/cdc/default/default/task/position/770fe66c-57af-4513-80b7-ee5a52ddabb9/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 + grep -q 'failed to get info:' {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {UpstreamID:7365782951458528569 Namespace:default ID:e26eb23a-fcbc-421f-ba32-40d6c4d6794b SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 15:20:02.463357053 +0800 CST StartTs:449571717726339078 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f08510 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571717752815623} {CheckpointTs:449571719862812682 MinTableBarrierTs:449571719993884708 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571719862812682, checkpointTs: 449571719862812682, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/770fe66c-57af-4513-80b7-ee5a52ddabb9 {"id":"770fe66c-57af-4513-80b7-ee5a52ddabb9","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc5cf6984 770fe66c-57af-4513-80b7-ee5a52ddabb9 /tidb/cdc/default/default/changefeed/info/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"upstream-id":7365782951458528569,"namespace":"default","changefeed-id":"e26eb23a-fcbc-421f-ba32-40d6c4d6794b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T15:20:02.463357053+08:00","start-ts":449571717726339078,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571717752815623} /tidb/cdc/default/default/changefeed/status/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":449571719862812682,"min-table-barrier-ts":449571719993884708,"admin-job-type":0} /tidb/cdc/default/default/task/position/770fe66c-57af-4513-80b7-ee5a52ddabb9/e26eb23a-fcbc-421f-ba32-40d6c4d6794b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782951458528569 {"id":7365782951458528569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:59 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/624fad67-f879-47bd-994f-befddc7b20e0 {"id":"624fad67-f879-47bd-994f-befddc7b20e0","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd569 624fad67-f879-47bd-994f-befddc7b20e0 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/624fad67-f879-47bd-994f-befddc7b20e0 {"id":"624fad67-f879-47bd-994f-befddc7b20e0","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd569 624fad67-f879-47bd-994f-befddc7b20e0 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/624fad67-f879-47bd-994f-befddc7b20e0 {"id":"624fad67-f879-47bd-994f-befddc7b20e0","address":"127.0.0.1:8302","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/capture/80195082-d3cc-46a4-aaeb-f58d83186da8 {"id":"80195082-d3cc-46a4-aaeb-f58d83186da8","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd53d 80195082-d3cc-46a4-aaeb-f58d83186da8 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6abd569 624fad67-f879-47bd-994f-befddc7b20e0 /tidb/cdc/default/default/upstream/7365783192926945855 {"id":7365783192926945855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 e090448f-dc5b-4b67-b802-eeb454f42a20 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=e090448f-dc5b-4b67-b802-eeb454f42a20 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c e090448f-dc5b-4b67-b802-eeb454f42a20 -s + info='{ "upstream_id": 7365783192926945855, "namespace": "default", "id": "e090448f-dc5b-4b67-b802-eeb454f42a20", "state": "normal", "checkpoint_tso": 449571730919522305, "checkpoint_time": "2024-05-06 15:20:52.641", "error": null }' + echo '{ "upstream_id": 7365783192926945855, "namespace": "default", "id": "e090448f-dc5b-4b67-b802-eeb454f42a20", "state": "normal", "checkpoint_tso": 449571730919522305, "checkpoint_time": "2024-05-06 15:20:52.641", "error": null }' { "upstream_id": 7365783192926945855, "namespace": "default", "id": "e090448f-dc5b-4b67-b802-eeb454f42a20", "state": "normal", "checkpoint_tso": 449571730919522305, "checkpoint_time": "2024-05-06 15:20:52.641", "error": null } ++ echo '{' '"upstream_id":' 7365783192926945855, '"namespace":' '"default",' '"id":' '"e090448f-dc5b-4b67-b802-eeb454f42a20",' '"state":' '"normal",' '"checkpoint_tso":' 449571730919522305, '"checkpoint_time":' '"2024-05-06' '15:20:52.641",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365783192926945855, '"namespace":' '"default",' '"id":' '"e090448f-dc5b-4b67-b802-eeb454f42a20",' '"state":' '"normal",' '"checkpoint_tso":' 449571730919522305, '"checkpoint_time":' '"2024-05-06' '15:20:52.641",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later [2024/05/06 15:21:00.065 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 15:21:00.078 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 15:21:00.079 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:20:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d29ebfd3-16fd-4698-9708-18f20bcb4c87 {"id":"d29ebfd3-16fd-4698-9708-18f20bcb4c87","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6aeff08 d29ebfd3-16fd-4698-9708-18f20bcb4c87 /tidb/cdc/default/default/upstream/7365783190183866646 {"id":7365783190183866646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d29ebfd3-16fd-4698-9708-18f20bcb4c87 {"id":"d29ebfd3-16fd-4698-9708-18f20bcb4c87","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6aeff08 d29ebfd3-16fd-4698-9708-18f20bcb4c87 /tidb/cdc/default/default/upstream/7365783190183866646 {"id":7365783190183866646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d29ebfd3-16fd-4698-9708-18f20bcb4c87 {"id":"d29ebfd3-16fd-4698-9708-18f20bcb4c87","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc6aeff08 d29ebfd3-16fd-4698-9708-18f20bcb4c87 /tidb/cdc/default/default/upstream/7365783190183866646 {"id":7365783190183866646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22702.out cli changefeed create --start-ts=449571731578028036 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 2e10d1cf-6dab-43bb-ba17-4006c4a6c555 Info: {"upstream_id":7365783190183866646,"namespace":"default","id":"2e10d1cf-6dab-43bb-ba17-4006c4a6c555","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:21:00.218483646+08:00","start_ts":449571731578028036,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571731578028036,"checkpoint_ts":449571731578028036,"checkpoint_time":"2024-05-06 15:20:55.153"} PASS nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass wait process cdc.test exit for 1-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 15:21:00.465 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 15:21:00.466 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 15:21:00.471 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 15:21:00.634 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 15:21:00.635 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 15:21:00.638 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:00 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> wait process cdc.test exit for 2-th time... [2024/05/06 15:21:00.800 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 15:21:00.812 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 15:21:00.813 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 15:21:00.967 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 15:21:00.968 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 15:21:00.981 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:01 CST 2024] <<<<<< run test case availability success! >>>>>> [2024/05/06 15:21:01.140 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 15:21:01.150 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 15:21:01.150 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 15:21:01.306 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 15:21:01.307 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 15:21:01.315 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 15:21:01.475 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 15:21:01.477 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 15:21:01.478 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 15:21:01.638 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 15:21:01.639 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 15:21:01.643 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] + set +x succeed to verify meta placement rules TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later table kill_owner_with_ddl.t3 exists check diff failed 1-th time, retry later table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:03 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later check diff successfully table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:05 CST 2024] <<<<<< run test case kill_owner_with_ddl success! >>>>>> table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later check data successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:06 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:07 CST 2024] <<<<<< run test case simple success! >>>>>> table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later [2024/05/06 15:21:07.602 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/06 15:21:07.800 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:21:07.804 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:21:07.962 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:21:07.963 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:21:07.971 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:21:07.973 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:21:08.138 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 15:21:08.147 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 15:21:08.381 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:21:08.383 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:21:08.469 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 15:21:08.471 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later [2024/05/06 15:21:08.797 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 15:21:08.806 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 15:21:08.971 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 15:21:08.973 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:21:08.974 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:21:08.976 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:09 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> [2024/05/06 15:21:09.146 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 15:21:09.149 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:21:09.318 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 15:21:09.318 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 15:21:09.320 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:21:09.320 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 15:21:09.485 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:21:09.493 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 15:21:09.644 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 15:21:09.646 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 15:21:09.666 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 15:21:09.668 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 15:21:09.812 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 15:21:09.832 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 15:21:09.978 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 15:21:09.979 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 15:21:09.997 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 15:21:10.000 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] table changefeed_auto_stop_4.usertable exists [2024/05/06 15:21:10.143 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 15:21:10.168 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 15:21:10.364 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 15:21:10.366 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] check diff failed 1-th time, retry later [2024/05/06 15:21:10.394 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 15:21:10.396 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 15:21:10.522 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 15:21:10.554 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] table consistent_replicate_storage_s3.t2 exists [2024/05/06 15:21:10.685 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 15:21:10.686 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 15:21:10.719 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 15:21:10.721 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 15:21:10.843 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 15:21:10.879 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] check diff successfully [2024/05/06 15:21:11.002 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 15:21:11.004 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 15:21:11.038 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 15:21:11.040 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + SINK_TYPE=mysql + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.67.208 + export UP_TIDB_HOST=10.233.67.208 UP_PD_HOST_1=10.233.67.208 UP_PD_HOST_2=10.233.67.208 UP_PD_HOST_3=10.233.67.208 UP_TIKV_HOST_1=10.233.67.208 UP_TIKV_HOST_2=10.233.67.208 UP_TIKV_HOST_3=10.233.67.208 + UP_TIDB_HOST=10.233.67.208 + UP_PD_HOST_1=10.233.67.208 + UP_PD_HOST_2=10.233.67.208 + UP_PD_HOST_3=10.233.67.208 + UP_TIKV_HOST_1=10.233.67.208 + UP_TIKV_HOST_2=10.233.67.208 + UP_TIKV_HOST_3=10.233.67.208 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=11911 + trap 'stop_tidb_cluster && stop_proxy' EXIT + '[' mysql == mysql ']' + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.67.208' UP_TIDB_HOST 10.233.67.208 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.67.208' UP_PD_HOST_1 10.233.67.208 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.67.208' UP_PD_HOST_2 10.233.67.208 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.67.208' UP_PD_HOST_3 10.233.67.208 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.67.208' UP_TIKV_HOST_1 10.233.67.208 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.67.208' UP_TIKV_HOST_2 10.233.67.208 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.67.208' UP_TIKV_HOST_3 10.233.67.208 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster [2024/05/06 15:21:11.163 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 15:21:11.199 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 15:21:11.326 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 15:21:11.328 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 15:21:11.365 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 15:21:11.367 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] wait process cdc.test exit for 1-th time... [2024/05/06 15:21:11.490 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 15:21:11.532 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 15:21:11.648 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 15:21:11.650 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] wait process cdc.test exit for 2-th time... [2024/05/06 15:21:11.692 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 15:21:11.694 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 15:21:11.810 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 15:21:11.855 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 15:21:11.968 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 15:21:11.970 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 15:21:12.013 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 15:21:12.014 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 15:21:12.125 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 15:21:12.170 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] wait process cdc.test exit for 3-th time... [2024/05/06 15:21:12.289 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 15:21:12.291 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 15:21:12.338 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 15:21:12.339 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:12 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> [2024/05/06 15:21:12.451 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 15:21:12.505 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:21:12 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1737817380.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:13 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:15 CST 2024] <<<<<< run test case multi_topics success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:21:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/0a2e270d-3f86-471c-b801-8bd335426c6e {UpstreamID:7365782998725739437 Namespace:default ID:0a2e270d-3f86-471c-b801-8bd335426c6e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:20:14.035498171 +0800 CST StartTs:449571720778743810 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dbf680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571720791851013} {CheckpointTs:449571722050404378 MinTableBarrierTs:449571736861540356 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690c1b8e-4a45-4264-94c4-2708b5f26582 {"id":"690c1b8e-4a45-4264-94c4-2708b5f26582","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc603231c 690c1b8e-4a45-4264-94c4-2708b5f26582 /tidb/cdc/default/default/changefeed/info/0a2e270d-3f86-471c-b801-8bd335426c6e {"upstream-id":7365782998725739437,"namespace":"default","changefeed-id":"0a2e270d-3f86-471c-b801-8bd335426c6e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:20:14.035498171+08:00","start-ts":449571720778743810,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571720791851013} /tidb/cdc/default/default/changefeed/status/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":449571722050404378,"min-table-barrier-ts":449571736861540356,"admin-job-type":0} /tidb/cdc/default/default/task/position/690c1b8e-4a45-4264-94c4-2708b5f26582/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0a2e270d-3f86-471c-b801-8bd335426c6e {UpstreamID:7365782998725739437 Namespace:default ID:0a2e270d-3f86-471c-b801-8bd335426c6e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:20:14.035498171 +0800 CST StartTs:449571720778743810 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dbf680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571720791851013} {CheckpointTs:449571722050404378 MinTableBarrierTs:449571736861540356 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690c1b8e-4a45-4264-94c4-2708b5f26582 {"id":"690c1b8e-4a45-4264-94c4-2708b5f26582","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc603231c 690c1b8e-4a45-4264-94c4-2708b5f26582 /tidb/cdc/default/default/changefeed/info/0a2e270d-3f86-471c-b801-8bd335426c6e {"upstream-id":7365782998725739437,"namespace":"default","changefeed-id":"0a2e270d-3f86-471c-b801-8bd335426c6e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:20:14.035498171+08:00","start-ts":449571720778743810,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571720791851013} /tidb/cdc/default/default/changefeed/status/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":449571722050404378,"min-table-barrier-ts":449571736861540356,"admin-job-type":0} /tidb/cdc/default/default/task/position/690c1b8e-4a45-4264-94c4-2708b5f26582/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/0a2e270d-3f86-471c-b801-8bd335426c6e {UpstreamID:7365782998725739437 Namespace:default ID:0a2e270d-3f86-471c-b801-8bd335426c6e SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:20:14.035498171 +0800 CST StartTs:449571720778743810 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003dbf680 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571720791851013} {CheckpointTs:449571722050404378 MinTableBarrierTs:449571736861540356 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:111,start_key:7480000000000000ff6f5f720000000000fa,end_key:7480000000000000ff6f5f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571722050404378, checkpointTs: 449571722050404378, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690c1b8e-4a45-4264-94c4-2708b5f26582 {"id":"690c1b8e-4a45-4264-94c4-2708b5f26582","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc603231c 690c1b8e-4a45-4264-94c4-2708b5f26582 /tidb/cdc/default/default/changefeed/info/0a2e270d-3f86-471c-b801-8bd335426c6e {"upstream-id":7365782998725739437,"namespace":"default","changefeed-id":"0a2e270d-3f86-471c-b801-8bd335426c6e","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:20:14.035498171+08:00","start-ts":449571720778743810,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571720791851013} /tidb/cdc/default/default/changefeed/status/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":449571722050404378,"min-table-barrier-ts":449571736861540356,"admin-job-type":0} /tidb/cdc/default/default/task/position/690c1b8e-4a45-4264-94c4-2708b5f26582/0a2e270d-3f86-471c-b801-8bd335426c6e {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365782998725739437 {"id":7365782998725739437,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:18 CST 2024] <<<<<< run test case avro_basic success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:21:18.569 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/06 15:21:18.784 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:21:18.789 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:21:19.084 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:21:19.085 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:21:19.086 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:21:19.088 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:21:19.287 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 15:21:19.287 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 15:21:19.449 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:21:19.449 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 15:21:19.450 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:21:19.451 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 15:21:19.618 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 15:21:19.618 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2024/05/06 15:21:20.071 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 15:21:20.072 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:21:20.073 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:21:20.074 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 15:21:20.238 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 15:21:20.238 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:21:20.402 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 15:21:20.404 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 15:21:20.404 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:21:20.406 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.67.208' (111) [2024/05/06 15:21:20.560 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:21:20.563 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 15:21:20.778 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 15:21:20.780 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 15:21:20.781 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 15:21:20.783 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:21:21.197 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 15:21:21.203 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 15:21:21.366 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 15:21:21.368 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 15:21:21.369 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 15:21:21.371 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:21 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> [2024/05/06 15:21:21.529 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 15:21:21.530 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:21:21.688 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 15:21:21.689 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 15:21:21.690 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 15:21:21.690 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 15:21:21.841 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 15:21:21.846 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 15:21:21.997 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 15:21:21.999 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 15:21:22.002 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 15:21:22.004 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 15:21:22.158 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 15:21:22.160 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 15:21:22.322 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 15:21:22.322 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 15:21:22.323 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 15:21:22.324 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 15:21:22.481 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 15:21:22.484 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 15:21:22.635 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 15:21:22.637 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 15:21:22.639 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 15:21:22.640 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.67.208' (111) [2024/05/06 15:21:22.788 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 15:21:22.794 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/06 15:21:22.951 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 15:21:22.952 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 15:21:22.953 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 15:21:22.954 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 15:21:23.104 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 15:21:23.108 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [2024/05/06 15:21:23.260 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 15:21:23.261 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 15:21:23.267 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 15:21:23.268 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 15:21:23.414 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:21:23.421 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 15:21:23.565 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 15:21:23.567 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.67.208' (111) [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:24 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d7e300007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:7092, start at 2024-05-06 15:21:25.651456188 +0800 CST m=+5.163780892 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:25.658 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:25.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:25.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d7e300007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:7092, start at 2024-05-06 15:21:25.651456188 +0800 CST m=+5.163780892 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:25.658 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:25.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:25.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d7e08000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:7178, start at 2024-05-06 15:21:25.646053946 +0800 CST m=+5.106811191 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:25.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:25.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:25.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["10.233.67.208:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d8498000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:11237, start at 2024-05-06 15:21:26.067049701 +0800 CST m=+5.094762397 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:26.074 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:26.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:26.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d8498000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:11237, start at 2024-05-06 15:21:26.067049701 +0800 CST m=+5.094762397 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:26.074 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:26.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:26.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331d847c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-1q55n, pid:11321, start at 2024-05-06 15:21:26.068467907 +0800 CST m=+5.044422823 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:26.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:26.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:26.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:27 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=8527 + echo 'proxy port: 11911' proxy port: 11911 + echo 'proxy pid: 8527' proxy pid: 8527 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=11911 go: downloading google.golang.org/grpc v1.62.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading golang.org/x/sys v0.19.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/rs/cors v1.7.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331da8b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:10092, start at 2024-05-06 15:21:28.401181836 +0800 CST m=+5.348630821 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:28.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:28.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:28.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12577.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 15:21:29.525 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/06 15:21:29.728 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 15:21:29.729 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 15:21:29.905 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 15:21:29.907 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 15:21:29.907 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 15:21:29.909 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 15:21:30.084 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 15:21:30.088 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:30 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> [2024/05/06 15:21:30.531 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 15:21:30.533 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 15:21:30.533 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 15:21:30.535 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 15:21:30.697 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 15:21:30.698 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:21:30.924 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 15:21:30.926 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 15:21:30.936 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 15:21:30.939 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] + set +x + tso='449571740596305921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571740596305921 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 15:21:31.270 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [Mon May 6 15:21:31 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1262712629.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331da8b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:10092, start at 2024-05-06 15:21:28.401181836 +0800 CST m=+5.348630821 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:28.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:28.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:28.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331da95c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-g2kvw, pid:10170, start at 2024-05-06 15:21:28.432668072 +0800 CST m=+5.311458282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:28.439 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:28.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:28.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 15:21:31.463 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 15:21:31.783 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 15:21:31.785 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 15:21:31.898 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 15:21:31.901 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 15:21:32.047 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 15:21:32.171 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 15:21:32.290 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 15:21:32.293 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 15:21:32.817 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 15:21:32.820 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 15:21:32.951 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 15:21:33.164 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 15:21:33.189 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 15:21:33.191 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 15:21:33.378 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 15:21:33.413 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 15:21:33.415 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:33 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> [2024/05/06 15:21:33.911 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 15:21:33.959 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [Mon May 6 15:21:33 CST 2024] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1147811480.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 15:21:34.076 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 15:21:34.294 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 15:21:34.476 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 15:21:34.480 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:21:34 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/9b689215-bb00-450b-b8c2-ab7bf64543ab {"id":"9b689215-bb00-450b-b8c2-ab7bf64543ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4cc737540c 9b689215-bb00-450b-b8c2-ab7bf64543ab /tidb/cdc/test1/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/9b689215-bb00-450b-b8c2-ab7bf64543ab {"id":"9b689215-bb00-450b-b8c2-ab7bf64543ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4cc737540c 9b689215-bb00-450b-b8c2-ab7bf64543ab /tidb/cdc/test1/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/9b689215-bb00-450b-b8c2-ab7bf64543ab {"id":"9b689215-bb00-450b-b8c2-ab7bf64543ab","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/22318f4cc737540c 9b689215-bb00-450b-b8c2-ab7bf64543ab /tidb/cdc/test1/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 15:21:34 CST 2024] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1267312675.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + export http_proxy=http://127.0.0.1:11911 + http_proxy=http://127.0.0.1:11911 + export https_proxy=http://127.0.0.1:11911 + https_proxy=http://127.0.0.1:11911 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.67.208:2379' try to connect pd cluster via proxy, pd addr: 10.233.67.208:2379 + ensure 10 curl http://10.233.67.208:2379/ curl http://10.233.67.208:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11911; Connection refused run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331df9940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:23775, start at 2024-05-06 15:21:33.568951382 +0800 CST m=+5.084738939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:33.577 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:33.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:33.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331df9940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:23775, start at 2024-05-06 15:21:33.568951382 +0800 CST m=+5.084738939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:33.577 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:33.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:33.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331dfbd80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:23860, start at 2024-05-06 15:21:33.717460244 +0800 CST m=+5.173762654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:33.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:33.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:33.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } curl http://10.233.67.208:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11911; Connection refused run task failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:21:36 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2513aeaa-121f-4c78-b096-30fe1e2e0c36 {"id":"2513aeaa-121f-4c78-b096-30fe1e2e0c36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc74184f3 2513aeaa-121f-4c78-b096-30fe1e2e0c36 /tidb/cdc/default/default/upstream/7365783354441770370 {"id":7365783354441770370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2513aeaa-121f-4c78-b096-30fe1e2e0c36 {"id":"2513aeaa-121f-4c78-b096-30fe1e2e0c36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc74184f3 2513aeaa-121f-4c78-b096-30fe1e2e0c36 /tidb/cdc/default/default/upstream/7365783354441770370 {"id":7365783354441770370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2513aeaa-121f-4c78-b096-30fe1e2e0c36 {"id":"2513aeaa-121f-4c78-b096-30fe1e2e0c36","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc74184f3 2513aeaa-121f-4c78-b096-30fe1e2e0c36 /tidb/cdc/default/default/upstream/7365783354441770370 {"id":7365783354441770370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Mon May 6 15:21:36 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.25159.out cli tso query --pd=http://127.0.0.1:2379 table owner_remove_table_error.finished_mark not exists for 1-th check, retry later + set +x + tso='449571742585192449 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571742585192449 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Mon May 6 15:21:38 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2520525207.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:21:38 GMT < Content-Length: 605 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/54dd9c16-14b3-4cc9-bc59-c98eaa55867b {"id":"54dd9c16-14b3-4cc9-bc59-c98eaa55867b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4cc7375427 54dd9c16-14b3-4cc9-bc59-c98eaa55867b /tidb/cdc/test2/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/54dd9c16-14b3-4cc9-bc59-c98eaa55867b {"id":"54dd9c16-14b3-4cc9-bc59-c98eaa55867b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4cc7375427 54dd9c16-14b3-4cc9-bc59-c98eaa55867b /tidb/cdc/test2/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/54dd9c16-14b3-4cc9-bc59-c98eaa55867b {"id":"54dd9c16-14b3-4cc9-bc59-c98eaa55867b","address":"127.0.0.1:8301","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/22318f4cc7375427 54dd9c16-14b3-4cc9-bc59-c98eaa55867b /tidb/cdc/test2/default/upstream/7365783342686545166 {"id":7365783342686545166,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12722.out cli changefeed create --start-ts=449571740596305921 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml Create changefeed successfully! ID: cf7fd0fc-36f4-4a18-9bde-49b4b0e03f3b Info: {"upstream_id":7365783342686545166,"namespace":"default","id":"cf7fd0fc-36f4-4a18-9bde-49b4b0e03f3b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:21:38.492923284+08:00","start_ts":449571740596305921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571740596305921,"checkpoint_ts":449571740596305921,"checkpoint_time":"2024-05-06 15:21:29.555"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 table owner_remove_table_error.finished_mark not exists for 2-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12758.out cli changefeed create --start-ts=449571740596305921 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml curl http://10.233.67.208:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:11911; Connection refused run task failed 3-th time, retry later Create changefeed successfully! ID: 4acfea51-1545-448a-83d8-8630b204041f Info: {"upstream_id":7365783342686545166,"namespace":"default","id":"4acfea51-1545-448a-83d8-8630b204041f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:21:40.268393409+08:00","start_ts":449571740596305921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571740596305921,"checkpoint_ts":449571740596305921,"checkpoint_time":"2024-05-06 15:21:29.555"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide time="2024-05-06T15:21:41+08:00" level=info msg="Listening on 127.0.0.1:11911" time="2024-05-06T15:21:41+08:00" level=info msg="Not intercepting TLS connections" + set +x * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCD550EEDD4BDA < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 07:21:41 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table owner_remove_table_error.finished_mark not exists for 3-th check, retry later table test.multi_cdc1 exists table test.multi_cdc2 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:21:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f37a4cde-432e-431d-849f-7f91561592b1 {"id":"f37a4cde-432e-431d-849f-7f91561592b1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc754c7ff f37a4cde-432e-431d-849f-7f91561592b1 /tidb/cdc/default/default/upstream/7365783378592366579 {"id":7365783378592366579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f37a4cde-432e-431d-849f-7f91561592b1 {"id":"f37a4cde-432e-431d-849f-7f91561592b1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc754c7ff f37a4cde-432e-431d-849f-7f91561592b1 /tidb/cdc/default/default/upstream/7365783378592366579 {"id":7365783378592366579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f37a4cde-432e-431d-849f-7f91561592b1 {"id":"f37a4cde-432e-431d-849f-7f91561592b1","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc754c7ff f37a4cde-432e-431d-849f-7f91561592b1 /tidb/cdc/default/default/upstream/7365783378592366579 {"id":7365783378592366579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_redo_resolved_ts 0a2e270d-3f86-471c-b801-8bd335426c6e 449571742917066755 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/0a2e270d-3f86-471c-b801-8bd335426c6e/meta global resolved ts 449571742368137226 not forward to 449571742917066755 run task failed 1-th time, retry later check_redo_resolved_ts 0a2e270d-3f86-471c-b801-8bd335426c6e 449571742917066755 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/0a2e270d-3f86-471c-b801-8bd335426c6e/meta global resolved ts 449571742890852356 not forward to 449571742917066755 run task failed 2-th time, retry later table owner_remove_table_error.finished_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... check_redo_resolved_ts 0a2e270d-3f86-471c-b801-8bd335426c6e 449571742917066755 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/0a2e270d-3f86-471c-b801-8bd335426c6e/meta run task successfully start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:45 CST 2024] <<<<<< run test case owner_remove_table_error success! >>>>>> [2024/05/06 15:21:42.152 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.613411829s] [2024/05/06 15:21:42.154 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:21:45 CST 2024] <<<<<< run test case multi_cdc_cluster success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... [2024/05/06 15:21:46.470 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m11.262061395s"] curl http://10.233.67.208:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 14937 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 8527 started proxy pid: 8527 + echo started proxy at port: 11911 started proxy at port: 11911 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.67.208 2379 + pd_host=10.233.67.208 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://10.233.67.208:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12246.out cli tso query --pd=http://10.233.67.208:2379 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 15:21:46.815 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 15:21:46.816 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:21:46.819 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 15:21:46.819 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449571737188958220] [resolvedTs=449571743939428356] [2024/05/06 15:21:46.820 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:21:46.820 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:21:46.820 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:21:46.821 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 15:21:46.825 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=5.983889ms] [2024/05/06 15:21:46.826 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:21:46.830 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 15:21:46.830 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 15:21:46.831 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 15:21:46.831 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 15:21:46.835 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 15:21:46.837 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 15:21:47.403 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=583.928315ms] [2024/05/06 15:21:47.404 +08:00] [ERROR] [redo.go:268] ["ignore unsupported DDL"] [ddl="{\"StartTs\":449571737188958214,\"CommitTs\":449571737188958220,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_storage_s3\",\"tbl-name\":\"t1\",\"tbl-id\":108,\"is-partition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... + set +x failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected + tso='449571744945012740 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571744945012740 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449571744945012740 + echo 'query start ts: 449571744945012740' query start ts: 449571744945012740 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Mon May 6 15:21:47 CST 2024] <<<<<< START cdc server in http_proxies case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.1228512287.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 11911 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 11911 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Mon, 06 May 2024 07:21:47 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled [2024/05/06 15:21:48.442 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571743939428356] [2024/05/06 15:21:48.442 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}] [checkpointTs=449571743939428356] [2024/05/06 15:21:48.184 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] [2024/05/06 15:21:48.543 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571743939428356] [2024/05/06 15:21:48.543 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}] [checkpointTs=449571743939428356] Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/05/06 15:21:49.044 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449571743939428356] [2024/05/06 15:21:49.044 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:118,start_key:7480000000000000ff765f720000000000fa,end_key:7480000000000000ff765f730000000000fa}] [checkpointTs=449571743939428356] [2024/05/06 15:21:49.144 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=449571743939428356] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 15:21:49.144 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 15:21:49.145 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] Apply redo log successfully wait process cdc.test exit for 1-th time... table mark.finish_mark not exists for 2-th check, retry later check diff successfully [Mon May 6 15:21:49 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to proxy 127.0.0.1 port 11911 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 11911 (#0) > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 15:21:50 CST 2024] <<<<<< run test case cdc success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Content-Length: 617 < Content-Type: text/plain; charset=utf-8 < Date: Mon, 06 May 2024 07:21:50 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aebf031f-bfd4-4340-9b66-b2220f87a578 {"id":"aebf031f-bfd4-4340-9b66-b2220f87a578","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/37558f4cc73b6412 aebf031f-bfd4-4340-9b66-b2220f87a578 /tidb/cdc/default/default/upstream/7365783346571299321 {"id":7365783346571299321,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.67.208:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aebf031f-bfd4-4340-9b66-b2220f87a578 {"id":"aebf031f-bfd4-4340-9b66-b2220f87a578","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/37558f4cc73b6412 aebf031f-bfd4-4340-9b66-b2220f87a578 /tidb/cdc/default/default/upstream/7365783346571299321 {"id":7365783346571299321,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.67.208:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aebf031f-bfd4-4340-9b66-b2220f87a578 {"id":"aebf031f-bfd4-4340-9b66-b2220f87a578","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/37558f4cc73b6412 aebf031f-bfd4-4340-9b66-b2220f87a578 /tidb/cdc/default/default/upstream/7365783346571299321 {"id":7365783346571299321,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.67.208:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=449571744945012740 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.12336.out cli changefeed create --start-ts=449571744945012740 --sink-uri=blackhole:/// failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled Create changefeed successfully! ID: 5e868443-15c8-4456-977b-9fd056f804ac Info: {"upstream_id":7365783346571299321,"namespace":"default","id":"5e868443-15c8-4456-977b-9fd056f804ac","sink_uri":"blackhole:///","create_time":"2024-05-06T15:21:51.083733653+08:00","start_ts":449571744945012740,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571744945012740,"checkpoint_ts":449571744945012740,"checkpoint_time":"2024-05-06 15:21:46.144"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark not exists for 3-th check, retry later + set +x + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table mark.finish_mark not exists for 5-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331f56340015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:34295, start at 2024-05-06 15:21:55.877627699 +0800 CST m=+5.210965492 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:55.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:55.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:55.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark not exists for 6-th check, retry later + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq xargs: unmatched double quote; by default quotes are special to xargs unless you use the -0 option + service_type_count=5 + echo 'captured services: ' captured services: + echo . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager '[2024/05' . /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager [2024/05 + '[' 5 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Mon May 6 15:21:57 CST 2024] <<<<<< run test case http_proxies success! >>>>>>' [Mon May 6 15:21:57 CST 2024] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Unavailable desc = client disconnected failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Unavailable desc = client disconnected VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331f56340015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:34295, start at 2024-05-06 15:21:55.877627699 +0800 CST m=+5.210965492 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:55.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:55.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:55.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d331f56cc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-kn26f, pid:34362, start at 2024-05-06 15:21:55.940243284 +0800 CST m=+5.223043160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:23:55.946 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:21:55.941 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:11:55.941 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark not exists for 7-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) table mark.finish_mark not exists for 8-th check, retry later [Mon May 6 15:22:01 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.3574035742.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table mark.finish_mark exists check diff successfully valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 15:22:01 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2529525297.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > wait process cdc.test exit for 1-th time... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/666c136d-4939-4b5b-9d5d-50324d3aefe6 {"id":"666c136d-4939-4b5b-9d5d-50324d3aefe6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7ab56f1 666c136d-4939-4b5b-9d5d-50324d3aefe6 /tidb/cdc/default/default/upstream/7365783471218601254 {"id":7365783471218601254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/666c136d-4939-4b5b-9d5d-50324d3aefe6 {"id":"666c136d-4939-4b5b-9d5d-50324d3aefe6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7ab56f1 666c136d-4939-4b5b-9d5d-50324d3aefe6 /tidb/cdc/default/default/upstream/7365783471218601254 {"id":7365783471218601254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/666c136d-4939-4b5b-9d5d-50324d3aefe6 {"id":"666c136d-4939-4b5b-9d5d-50324d3aefe6","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7ab56f1 666c136d-4939-4b5b-9d5d-50324d3aefe6 /tidb/cdc/default/default/upstream/7365783471218601254 {"id":7365783471218601254,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365783471218601254,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:04.309751516+08:00","start_ts":449571748853317633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571748853317633,"checkpoint_ts":449571748853317633,"checkpoint_time":"2024-05-06 15:22:01.053"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365783471218601254,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:04.505790223+08:00","start_ts":449571748853317633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571748853317633,"checkpoint_ts":449571748853317633,"checkpoint_time":"2024-05-06 15:22:01.053"} wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365783471218601254,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:04.655155441+08:00","start_ts":449571748853317633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571748853317633,"checkpoint_ts":449571748853317633,"checkpoint_time":"2024-05-06 15:22:01.053"} Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:22:05 CST 2024] <<<<<< run test case multi_source success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table multi_tables_ddl_test.t55 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + stop_proxy + kill 8527 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table multi_tables_ddl_test.t55 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table multi_tables_ddl_test.t55 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table multi_tables_ddl_test.t55 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33206bfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:18636, start at 2024-05-06 15:22:13.658517072 +0800 CST m=+5.394367957 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:13.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:13.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:13.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d332077300018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:23640, start at 2024-05-06 15:22:14.380801306 +0800 CST m=+5.004674274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:14.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:14.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:14.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d332077300018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:23640, start at 2024-05-06 15:22:14.380801306 +0800 CST m=+5.004674274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:14.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:14.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:14.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3320796c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-rx0m5, pid:23718, start at 2024-05-06 15:22:14.539789963 +0800 CST m=+5.089405882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:14.546 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:14.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:14.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33206bfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:18636, start at 2024-05-06 15:22:13.658517072 +0800 CST m=+5.394367957 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:13.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:13.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:13.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3320725c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-qlm32, pid:18728, start at 2024-05-06 15:22:14.074133906 +0800 CST m=+5.745312447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:14.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:14.089 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:14.089 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.25031.out cli tso query --pd=http://127.0.0.1:2379 [Mon May 6 15:22:18 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2005920061.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449571753334669314 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571753334669314 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 15:22:19 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2507425076.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449571750609682460, "checkpoint_time": "2024-05-06 15:22:07.753", "error": null }' + echo '{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449571750609682460, "checkpoint_time": "2024-05-06 15:22:07.753", "error": null }' { "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449571750609682460, "checkpoint_time": "2024-05-06 15:22:07.753", "error": null } ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449571750609682460, '"checkpoint_time":' '"2024-05-06' '15:22:07.753",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449571750609682460, '"checkpoint_time":' '"2024-05-06' '15:22:07.753",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449571753742303239, "checkpoint_time": "2024-05-06 15:22:19.703", "error": null }' + echo '{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449571753742303239, "checkpoint_time": "2024-05-06 15:22:19.703", "error": null }' { "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449571753742303239, "checkpoint_time": "2024-05-06 15:22:19.703", "error": null } ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449571753742303239, '"checkpoint_time":' '"2024-05-06' '15:22:19.703",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449571753742303239, '"checkpoint_time":' '"2024-05-06' '15:22:19.703",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449571750111608879, "checkpoint_time": "2024-05-06 15:22:05.853", "error": { "time": "2024-05-06T15:22:07.86770135+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449571750583468044]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449571750111608879, "checkpoint_time": "2024-05-06 15:22:05.853", "error": { "time": "2024-05-06T15:22:07.86770135+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449571750583468044]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365783471218601254, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449571750111608879, "checkpoint_time": "2024-05-06 15:22:05.853", "error": { "time": "2024-05-06T15:22:07.86770135+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449571750583468044]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449571750111608879, '"checkpoint_time":' '"2024-05-06' '15:22:05.853",' '"error":' '{' '"time":' '"2024-05-06T15:22:07.86770135+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449571750583468044]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365783471218601254, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449571750111608879, '"checkpoint_time":' '"2024-05-06' '15:22:05.853",' '"error":' '{' '"time":' '"2024-05-06T15:22:07.86770135+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449571750583468044]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449571750583468044]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-06 15:22:07.653 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449571750583468044. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449571750583468044]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fd25947-06d2-4dd8-b65c-2024aeaaa764 {"id":"7fd25947-06d2-4dd8-b65c-2024aeaaa764","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41af1 7fd25947-06d2-4dd8-b65c-2024aeaaa764 /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fd25947-06d2-4dd8-b65c-2024aeaaa764 {"id":"7fd25947-06d2-4dd8-b65c-2024aeaaa764","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41af1 7fd25947-06d2-4dd8-b65c-2024aeaaa764 /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fd25947-06d2-4dd8-b65c-2024aeaaa764 {"id":"7fd25947-06d2-4dd8-b65c-2024aeaaa764","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41af1 7fd25947-06d2-4dd8-b65c-2024aeaaa764 /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Mon May 6 15:22:21 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:22:22 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT table partition_table2.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3320e4880006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:13340, start at 2024-05-06 15:22:21.35297326 +0800 CST m=+5.088318664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:21.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:21.346 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:21.346 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3320e4880006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:13340, start at 2024-05-06 15:22:21.35297326 +0800 CST m=+5.088318664 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:21.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:21.346 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:21.346 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3320e5280008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-62f5v, pid:13423, start at 2024-05-06 15:22:21.393444083 +0800 CST m=+5.080682188 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:21.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:21.386 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:21.386 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sequence/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:22 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf0e1e6-5cd5-452c-aa94-56fa535bb00b {"id":"aaf0e1e6-5cd5-452c-aa94-56fa535bb00b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f26c11 aaf0e1e6-5cd5-452c-aa94-56fa535bb00b /tidb/cdc/default/default/upstream/7365783543060319351 {"id":7365783543060319351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf0e1e6-5cd5-452c-aa94-56fa535bb00b {"id":"aaf0e1e6-5cd5-452c-aa94-56fa535bb00b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f26c11 aaf0e1e6-5cd5-452c-aa94-56fa535bb00b /tidb/cdc/default/default/upstream/7365783543060319351 {"id":7365783543060319351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf0e1e6-5cd5-452c-aa94-56fa535bb00b {"id":"aaf0e1e6-5cd5-452c-aa94-56fa535bb00b","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f26c11 aaf0e1e6-5cd5-452c-aa94-56fa535bb00b /tidb/cdc/default/default/upstream/7365783543060319351 {"id":7365783543060319351,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.25134.out cli changefeed create --start-ts=449571753334669314 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: b03a7c9b-2921-4bcd-ab33-8436519253fe Info: {"upstream_id":7365783543060319351,"namespace":"default","id":"b03a7c9b-2921-4bcd-ab33-8436519253fe","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:23.260461458+08:00","start_ts":449571753334669314,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571753334669314,"checkpoint_ts":449571753334669314,"checkpoint_time":"2024-05-06 15:22:18.148"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table split_region.test1 not exists for 1-th check, retry later table partition_table2.t2 not exists for 2-th check, retry later [Mon May 6 15:22:24 CST 2024] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.1468014682.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table partition_table2.t2 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e72728b-e4dc-46c7-bb04-bc90223664af {"id":"1e72728b-e4dc-46c7-bb04-bc90223664af","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc8133af3 1e72728b-e4dc-46c7-bb04-bc90223664af /tidb/cdc/default/default/upstream/7365783581217861229 {"id":7365783581217861229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e72728b-e4dc-46c7-bb04-bc90223664af {"id":"1e72728b-e4dc-46c7-bb04-bc90223664af","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc8133af3 1e72728b-e4dc-46c7-bb04-bc90223664af /tidb/cdc/default/default/upstream/7365783581217861229 {"id":7365783581217861229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e72728b-e4dc-46c7-bb04-bc90223664af {"id":"1e72728b-e4dc-46c7-bb04-bc90223664af","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc8133af3 1e72728b-e4dc-46c7-bb04-bc90223664af /tidb/cdc/default/default/upstream/7365783581217861229 {"id":7365783581217861229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.14731.out cli changefeed create --start-ts=449571755024973825 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: 77ccfabc-3a4e-4831-8726-d631403bb1df Info: {"upstream_id":7365783581217861229,"namespace":"default","id":"77ccfabc-3a4e-4831-8726-d631403bb1df","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:28.123889146+08:00","start_ts":449571755024973825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571755024973825,"checkpoint_ts":449571755024973825,"checkpoint_time":"2024-05-06 15:22:24.596"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table partition_table2.t2 not exists for 4-th check, retry later table split_region.test2 exists check diff successfully + set +x table partition_table2.t2 not exists for 5-th check, retry later table sequence_test.t1 not exists for 1-th check, retry later check_changefeed_state http://127.0.0.1:2379 c9ab971e-8b36-413b-8c81-d7ae5e233124 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=c9ab971e-8b36-413b-8c81-d7ae5e233124 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c9ab971e-8b36-413b-8c81-d7ae5e233124 -s + info='{ "upstream_id": 7365783164781034075, "namespace": "default", "id": "c9ab971e-8b36-413b-8c81-d7ae5e233124", "state": "finished", "checkpoint_tso": 449571753790799874, "checkpoint_time": "2024-05-06 15:22:19.888", "error": null }' + echo '{ "upstream_id": 7365783164781034075, "namespace": "default", "id": "c9ab971e-8b36-413b-8c81-d7ae5e233124", "state": "finished", "checkpoint_tso": 449571753790799874, "checkpoint_time": "2024-05-06 15:22:19.888", "error": null }' { "upstream_id": 7365783164781034075, "namespace": "default", "id": "c9ab971e-8b36-413b-8c81-d7ae5e233124", "state": "finished", "checkpoint_tso": 449571753790799874, "checkpoint_time": "2024-05-06 15:22:19.888", "error": null } ++ echo '{' '"upstream_id":' 7365783164781034075, '"namespace":' '"default",' '"id":' '"c9ab971e-8b36-413b-8c81-d7ae5e233124",' '"state":' '"finished",' '"checkpoint_tso":' 449571753790799874, '"checkpoint_time":' '"2024-05-06' '15:22:19.888",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365783164781034075, '"namespace":' '"default",' '"id":' '"c9ab971e-8b36-413b-8c81-d7ae5e233124",' '"state":' '"finished",' '"checkpoint_tso":' 449571753790799874, '"checkpoint_time":' '"2024-05-06' '15:22:19.888",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:22:26 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table partition_table2.t2 not exists for 6-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table2.t2 not exists for 7-th check, retry later check diff failed 2-th time, retry later table partition_table2.t2 not exists for 8-th check, retry later check diff failed 3-th time, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2708 0 --:--:-- --:--:-- --:--:-- 2695 100 221 100 221 0 0 2704 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 15:22:25.294","puller_resolved_ts":"2024-05-06 15:22:17.293","last_synced_ts":"2024-05-06 15:20:08.993","now_ts":"2024-05-06 15:22:26.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '15:22:25.294","puller_resolved_ts":"2024-05-06' '15:22:17.293","last_synced_ts":"2024-05-06' '15:20:08.993","now_ts":"2024-05-06' '15:22:26.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 14887 9.6 0.0 14084392 148044 ? Sl 15:19 0:15 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 14945 6.0 0.0 14158060 136920 ? Sl 15:19 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714980151.7408772,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000da2fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714980151.740988,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714980151.8353405,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00087c8c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714980151.8354635,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714980152.3629317,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0024001c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714980152.3629918,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T15:22:36.177548+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:36.183278+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:36.28647+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table partition_table2.t2 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 4-th time, retry later table partition_table2.t2 exists check diff failed 1-th time, retry later check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release {"level":"warn","ts":"2024-05-06T15:22:42.179064+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:42.185158+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:42.287731+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff successfully check diff failed 6-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:22:44 CST 2024] <<<<<< run test case split_region success! >>>>>> Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 7-th time, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:22:47 CST 2024] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2024020242.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 8-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:22:48.18058+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:48.186085+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:48.288575+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 9-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/3e7f92fa-7000-492c-bc3b-6b2bd314937d {UpstreamID:7365783551471686832 Namespace:default ID:3e7f92fa-7000-492c-bc3b-6b2bd314937d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:22:21.904296136 +0800 CST StartTs:449571754300145666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bdb290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571754313252869} {CheckpointTs:449571761522212867 MinTableBarrierTs:449571761522212872 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:117,start_key:7480000000000000ff755f720000000000fa,end_key:7480000000000000ff755f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571761522212871, checkpointTs: 449571761522212867, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571761522212867, checkpointTs: 449571761522212867, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:105,start_key:7480000000000000ff695f720000000000fa,end_key:7480000000000000ff695f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:123,start_key:7480000000000000ff7b5f720000000000fa,end_key:7480000000000000ff7b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba942a20-0474-4229-8af2-9dd9471637ee {"id":"ba942a20-0474-4229-8af2-9dd9471637ee","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41bc8 ba942a20-0474-4229-8af2-9dd9471637ee /tidb/cdc/default/default/changefeed/info/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"upstream-id":7365783551471686832,"namespace":"default","changefeed-id":"3e7f92fa-7000-492c-bc3b-6b2bd314937d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:22:21.904296136+08:00","start-ts":449571754300145666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571754313252869} /tidb/cdc/default/default/changefeed/status/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":449571761522212867,"min-table-barrier-ts":449571761784356869,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba942a20-0474-4229-8af2-9dd9471637ee/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3e7f92fa-7000-492c-bc3b-6b2bd314937d {UpstreamID:7365783551471686832 Namespace:default ID:3e7f92fa-7000-492c-bc3b-6b2bd314937d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:22:21.904296136 +0800 CST StartTs:449571754300145666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bdb290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571754313252869} {CheckpointTs:449571761522212867 MinTableBarrierTs:449571761522212872 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:117,start_key:7480000000000000ff755f720000000000fa,end_key:7480000000000000ff755f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571761522212871, checkpointTs: 449571761522212867, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571761522212867, checkpointTs: 449571761522212867, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:105,start_key:7480000000000000ff695f720000000000fa,end_key:7480000000000000ff695f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:123,start_key:7480000000000000ff7b5f720000000000fa,end_key:7480000000000000ff7b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba942a20-0474-4229-8af2-9dd9471637ee {"id":"ba942a20-0474-4229-8af2-9dd9471637ee","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41bc8 ba942a20-0474-4229-8af2-9dd9471637ee /tidb/cdc/default/default/changefeed/info/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"upstream-id":7365783551471686832,"namespace":"default","changefeed-id":"3e7f92fa-7000-492c-bc3b-6b2bd314937d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:22:21.904296136+08:00","start-ts":449571754300145666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571754313252869} /tidb/cdc/default/default/changefeed/status/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":449571761522212867,"min-table-barrier-ts":449571761784356869,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba942a20-0474-4229-8af2-9dd9471637ee/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/3e7f92fa-7000-492c-bc3b-6b2bd314937d {UpstreamID:7365783551471686832 Namespace:default ID:3e7f92fa-7000-492c-bc3b-6b2bd314937d SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 15:22:21.904296136 +0800 CST StartTs:449571754300145666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bdb290 State:normal Error: Warning: CreatorVersion:v7.5.1-23-gf21441d0a Epoch:449571754313252869} {CheckpointTs:449571761522212867 MinTableBarrierTs:449571761522212872 AdminJobType:noop} span: {table_id:116,start_key:7480000000000000ff745f720000000000fa,end_key:7480000000000000ff745f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:107,start_key:7480000000000000ff6b5f720000000000fa,end_key:7480000000000000ff6b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:117,start_key:7480000000000000ff755f720000000000fa,end_key:7480000000000000ff755f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:109,start_key:7480000000000000ff6d5f720000000000fa,end_key:7480000000000000ff6d5f730000000000fa}, resolvedTs: 449571761522212871, checkpointTs: 449571761522212867, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449571761522212867, checkpointTs: 449571761522212867, state: Replicating span: {table_id:114,start_key:7480000000000000ff725f720000000000fa,end_key:7480000000000000ff725f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:119,start_key:7480000000000000ff775f720000000000fa,end_key:7480000000000000ff775f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:113,start_key:7480000000000000ff715f720000000000fa,end_key:7480000000000000ff715f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:105,start_key:7480000000000000ff695f720000000000fa,end_key:7480000000000000ff695f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating span: {table_id:123,start_key:7480000000000000ff7b5f720000000000fa,end_key:7480000000000000ff7b5f730000000000fa}, resolvedTs: 449571761522212872, checkpointTs: 449571761522212867, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ba942a20-0474-4229-8af2-9dd9471637ee {"id":"ba942a20-0474-4229-8af2-9dd9471637ee","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc7f41bc8 ba942a20-0474-4229-8af2-9dd9471637ee /tidb/cdc/default/default/changefeed/info/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"upstream-id":7365783551471686832,"namespace":"default","changefeed-id":"3e7f92fa-7000-492c-bc3b-6b2bd314937d","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T15:22:21.904296136+08:00","start-ts":449571754300145666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-23-gf21441d0a","epoch":449571754313252869} /tidb/cdc/default/default/changefeed/status/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":449571761522212867,"min-table-barrier-ts":449571761784356869,"admin-job-type":0} /tidb/cdc/default/default/task/position/ba942a20-0474-4229-8af2-9dd9471637ee/3e7f92fa-7000-492c-bc3b-6b2bd314937d {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365783551471686832 {"id":7365783551471686832,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33229eec0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:26226, start at 2024-05-06 15:22:49.688388597 +0800 CST m=+5.091908980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:49.695 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:49.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:49.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33229eec0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:26226, start at 2024-05-06 15:22:49.688388597 +0800 CST m=+5.091908980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:49.695 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:49.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:49.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33229f980004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-wtjxw, pid:26316, start at 2024-05-06 15:22:49.705250943 +0800 CST m=+5.058416059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:49.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:49.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:49.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 10-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 15:22:52 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2763927641.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 11-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:22:54.18149+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:54.18703+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:22:54.289747+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3322d9100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:6704, start at 2024-05-06 15:22:53.409000362 +0800 CST m=+5.067624569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:53.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:53.380 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:53.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3322d9100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:6704, start at 2024-05-06 15:22:53.409000362 +0800 CST m=+5.067624569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:53.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:53.380 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:53.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3322da6c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-glgzk, pid:6783, start at 2024-05-06 15:22:53.473259431 +0800 CST m=+5.058846258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:24:53.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:22:53.467 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:12:53.467 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9262aadb-b5ef-407b-840c-12e3646da619 {"id":"9262aadb-b5ef-407b-840c-12e3646da619","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc87e1af0 9262aadb-b5ef-407b-840c-12e3646da619 /tidb/cdc/default/default/upstream/7365783705144872177 {"id":7365783705144872177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9262aadb-b5ef-407b-840c-12e3646da619 {"id":"9262aadb-b5ef-407b-840c-12e3646da619","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc87e1af0 9262aadb-b5ef-407b-840c-12e3646da619 /tidb/cdc/default/default/upstream/7365783705144872177 {"id":7365783705144872177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9262aadb-b5ef-407b-840c-12e3646da619 {"id":"9262aadb-b5ef-407b-840c-12e3646da619","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc87e1af0 9262aadb-b5ef-407b-840c-12e3646da619 /tidb/cdc/default/default/upstream/7365783705144872177 {"id":7365783705144872177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.27689.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml check diff failed 12-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:22:56.169683+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T15:22:56.169738+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T15:22:56.174755+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T15:22:56.174822+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T15:22:56.277658+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T15:22:56.277696+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Create changefeed successfully! ID: 45155642-2d9f-4c07-aaf6-11a6dffd5c62 Info: {"upstream_id":7365783705144872177,"namespace":"default","id":"45155642-2d9f-4c07-aaf6-11a6dffd5c62","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T15:22:56.378399062+08:00","start_ts":449571763325239299,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571763325239299,"checkpoint_ts":449571763325239299,"checkpoint_time":"2024-05-06 15:22:56.259"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Mon May 6 15:22:56 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.80678069.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check diff failed 13-th time, retry later table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } check diff failed 14-th time, retry later [Pipeline] // withCredentials 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:23:00.182634+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:00.188212+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:00.29188+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:22:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229bca71-7c0c-4331-844a-271a6bc34b79 {"id":"229bca71-7c0c-4331-844a-271a6bc34b79","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc88b45ef 229bca71-7c0c-4331-844a-271a6bc34b79 /tidb/cdc/default/default/upstream/7365783714107082010 {"id":7365783714107082010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229bca71-7c0c-4331-844a-271a6bc34b79 {"id":"229bca71-7c0c-4331-844a-271a6bc34b79","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc88b45ef 229bca71-7c0c-4331-844a-271a6bc34b79 /tidb/cdc/default/default/upstream/7365783714107082010 {"id":7365783714107082010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/229bca71-7c0c-4331-844a-271a6bc34b79 {"id":"229bca71-7c0c-4331-844a-271a6bc34b79","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc88b45ef 229bca71-7c0c-4331-844a-271a6bc34b79 /tidb/cdc/default/default/upstream/7365783714107082010 {"id":7365783714107082010,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 448a5855-c90d-47a6-b02d-71122ae12f64 Info: {"upstream_id":7365783714107082010,"namespace":"default","id":"448a5855-c90d-47a6-b02d-71122ae12f64","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T15:23:00.042768772+08:00","start_ts":449571763461816321,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571763461816321,"checkpoint_ts":449571763461816321,"checkpoint_time":"2024-05-06 15:22:56.780"} [Pipeline] // timeout [Pipeline] } table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } check diff failed 15-th time, retry later [Pipeline] // stage [Pipeline] } table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later check diff failed 16-th time, retry later table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:23:06.183831+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:06.18934+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:06.29264+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff failed 17-th time, retry later table event_filter.t_alter exists 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714980186.7423978,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000da2fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714980186.742457,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later {"level":"warn","ts":1714980186.836594,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00087c8c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714980186.8366566,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714980187.3644364,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0024001c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714980187.3644876,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} check diff failed 18-th time, retry later table event_filter.finish_mark exists check diff failed 1-th time, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later check diff failed 2-th time, retry later check diff failed 19-th time, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later check diff failed 20-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-06T15:23:12.184658+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e58c40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:12.191119+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000efe1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T15:23:12.293101+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0007b76c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff successfully wait process cdc.test exit for 1-th time... table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:23:14 CST 2024] <<<<<< run test case event_filter success! >>>>>> check diff failed 21-th time, retry later table force_replicate_table.t6 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test check diff failed 22-th time, retry later wait process cdc.test exit for 1-th time... table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 2-th time... run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 check diff failed 23-th time, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 check diff failed 24-th time, retry later run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later check diff failed 25-th time, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 run task successfully wait process cdc.test exit for 1-th time... check diff failed 26-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 15:23:25 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 27-th time, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 28-th time, retry later check diff failed 29-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 30-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 31-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 32-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-361/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 33-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33259d540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:18230, start at 2024-05-06 15:23:38.741397898 +0800 CST m=+5.139155778 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:25:38.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:23:38.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:13:38.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33259d540013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:18230, start at 2024-05-06 15:23:38.741397898 +0800 CST m=+5.139155778 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:25:38.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:23:38.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:13:38.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d33259ec00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-361-q5lt7, pid:18308, start at 2024-05-06 15:23:38.830697093 +0800 CST m=+5.164871753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-15:25:38.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-15:23:38.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-15:13:38.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 34-th time, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19542.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 35-th time, retry later + set +x + tso='449571775371018241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449571775371018241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449571775371018241 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 15:23:43 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1957019572.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 36-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 07:23:46 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a1876a-c020-4ec0-bed8-a4e94bdc1928 {"id":"a4a1876a-c020-4ec0-bed8-a4e94bdc1928","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc93d84f6 a4a1876a-c020-4ec0-bed8-a4e94bdc1928 /tidb/cdc/default/default/upstream/7365783916904134213 {"id":7365783916904134213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a1876a-c020-4ec0-bed8-a4e94bdc1928 {"id":"a4a1876a-c020-4ec0-bed8-a4e94bdc1928","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc93d84f6 a4a1876a-c020-4ec0-bed8-a4e94bdc1928 /tidb/cdc/default/default/upstream/7365783916904134213 {"id":7365783916904134213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a4a1876a-c020-4ec0-bed8-a4e94bdc1928 {"id":"a4a1876a-c020-4ec0-bed8-a4e94bdc1928","address":"127.0.0.1:8300","version":"v7.5.1-23-gf21441d0a"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4cc93d84f6 a4a1876a-c020-4ec0-bed8-a4e94bdc1928 /tidb/cdc/default/default/upstream/7365783916904134213 {"id":7365783916904134213,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449571775371018241 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19625.out cli changefeed create --start-ts=449571775371018241 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365783916904134213,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T15:23:47.091791538+08:00","start_ts":449571775371018241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-23-gf21441d0a","resolved_ts":449571775371018241,"checkpoint_ts":449571775371018241,"checkpoint_time":"2024-05-06 15:23:42.210"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 37-th time, retry later + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 check diff failed 38-th time, retry later check diff failed 39-th time, retry later check diff failed 40-th time, retry later + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 17608 19.4 0.4 3766240 1617040 ? Sl 15:23 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 17609 19.6 0.4 3771364 1631644 ? Sl 15:23 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 17610 27.5 0.4 3811812 1685516 ? Sl 15:23 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 17612 26.5 0.4 3813860 1675360 ? Sl 15:23 0:05 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2938 0 --:--:-- --:--:-- --:--:-- 2963 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 15:23:52.660","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-06 15:23:49.009","now_ts":"2024-05-06 15:23:54.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '15:23:52.660","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-06' '15:23:49.009","now_ts":"2024-05-06' '15:23:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '15:23:52.660","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-06' '15:23:49.009","now_ts":"2024-05-06' '15:23:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 check diff failed 41-th time, retry later check diff failed 42-th time, retry later check diff failed 43-th time, retry later check diff failed 44-th time, retry later check diff failed 45-th time, retry later check diff failed 46-th time, retry later check diff failed 47-th time, retry later check diff failed 48-th time, retry later check diff failed 49-th time, retry later check diff failed 50-th time, retry later check diff failed 51-th time, retry later check diff failed 52-th time, retry later check diff failed 53-th time, retry later check diff failed 54-th time, retry later check diff failed 55-th time, retry later check diff failed 56-th time, retry later check diff failed 57-th time, retry later check diff failed 58-th time, retry later check diff failed 59-th time, retry later check diff failed 60-th time, retry later check diff failed at last There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log There is something error when initialize diff, please check log info in /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log [2024/05/06 15:24:34.573 +08:00] [INFO] [printer.go:46] ["Welcome to sync_diff_inspector"] ["Release Version"=v7.4.0] ["Git Commit Hash"=d671b0840063bc2532941f02e02e12627402844c] ["Git Branch"=heads/refs/tags/v7.4.0] ["UTC Build Time"="2023-09-22 03:51:56"] ["Go Version"=go1.21.1] [2024/05/06 15:24:34.574 +08:00] [INFO] [main.go:101] [config="{\"check-thread-count\":4,\"split-thread-count\":5,\"export-fix-sql\":true,\"check-struct-only\":false,\"dm-addr\":\"\",\"dm-task\":\"\",\"data-sources\":{\"mysql1\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null},\"tidb0\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null}},\"routes\":null,\"table-configs\":null,\"task\":{\"source-instances\":[\"mysql1\"],\"source-routes\":null,\"target-instance\":\"tidb0\",\"target-check-tables\":[\"sequence_test.t1\"],\"target-configs\":null,\"output-dir\":\"/tmp/tidb_cdc_test/sequence/sync_diff/output\",\"SourceInstances\":[{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null}],\"TargetInstance\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null},\"TargetTableConfigs\":null,\"TargetCheckTables\":[{}],\"FixDir\":\"/tmp/tidb_cdc_test/sequence/sync_diff/output/fix-on-tidb0\",\"CheckpointDir\":\"/tmp/tidb_cdc_test/sequence/sync_diff/output/checkpoint\",\"HashFile\":\"\"},\"ConfigFile\":\"/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/conf/diff_config.toml\",\"PrintVersion\":false}"] [2024/05/06 15:24:34.574 +08:00] [DEBUG] [diff.go:842] ["set tidb cfg"] [2024/05/06 15:24:34.576 +08:00] [DEBUG] [common.go:386] ["query tables"] [query="SHOW FULL TABLES IN `sequence_test` WHERE Table_Type = 'BASE TABLE';"] [2024/05/06 15:24:34.577 +08:00] [DEBUG] [common.go:386] ["query tables"] [query="SHOW FULL TABLES IN `test` WHERE Table_Type = 'BASE TABLE';"] [2024/05/06 15:24:34.577 +08:00] [DEBUG] [source.go:326] ["match target table"] [table=`sequence_test`.`t1`] [2024/05/06 15:24:34.578 +08:00] [FATAL] [main.go:120] ["failed to initialize diff process"] [error="get table sequence_test.t1's information error line 3 column 31 near \"nextval(`sequence_test`.`seq0`)),\n PRIMARY KEY (`id`) /*T![clustered_index] NONCLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\" \ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\ngithub.com/pingcap/tidb/parser.(*Parser).ParseSQL\n\t/go/pkg/mod/github.com/pingcap/tidb/parser@v0.0.0-20230823131104-05aa17143df8/yy_parser.go:170\ngithub.com/pingcap/tidb/parser.(*Parser).ParseOneStmt\n\t/go/pkg/mod/github.com/pingcap/tidb/parser@v0.0.0-20230823131104-05aa17143df8/yy_parser.go:191\ngithub.com/pingcap/tidb-tools/pkg/dbutil.getTableInfoBySQL\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:149\ngithub.com/pingcap/tidb-tools/pkg/dbutil.GetTableInfoBySQLWithSessionContext\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:140\ngithub.com/pingcap/tidb-tools/pkg/dbutil.GetTableInfoWithVersion\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:121\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.initTables\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:328\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.NewSources\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:121\nmain.(*Diff).init\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:137\nmain.NewDiff\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:95\nmain.checkSyncState\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:117\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:104\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"] [errorVerbose="get table sequence_test.t1's information error line 3 column 31 near \"nextval(`sequence_test`.`seq0`)),\n PRIMARY KEY (`id`) /*T![clustered_index] NONCLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin\" \ngithub.com/pingcap/errors.AddStack\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\ngithub.com/pingcap/errors.Trace\n\t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\ngithub.com/pingcap/tidb/parser.(*Parser).ParseSQL\n\t/go/pkg/mod/github.com/pingcap/tidb/parser@v0.0.0-20230823131104-05aa17143df8/yy_parser.go:170\ngithub.com/pingcap/tidb/parser.(*Parser).ParseOneStmt\n\t/go/pkg/mod/github.com/pingcap/tidb/parser@v0.0.0-20230823131104-05aa17143df8/yy_parser.go:191\ngithub.com/pingcap/tidb-tools/pkg/dbutil.getTableInfoBySQL\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:149\ngithub.com/pingcap/tidb-tools/pkg/dbutil.GetTableInfoBySQLWithSessionContext\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:140\ngithub.com/pingcap/tidb-tools/pkg/dbutil.GetTableInfoWithVersion\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/pkg/dbutil/table.go:121\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.initTables\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:328\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.NewSources\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:121\nmain.(*Diff).init\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:137\nmain.NewDiff\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:95\nmain.checkSyncState\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:117\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:104\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.initTables\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:330\ngithub.com/pingcap/tidb-tools/sync_diff_inspector/source.NewSources\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/source/source.go:121\nmain.(*Diff).init\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:137\nmain.NewDiff\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/diff.go:95\nmain.checkSyncState\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:117\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:104\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"] [stack="main.checkSyncState\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:120\nmain.main\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb-tools/sync_diff_inspector/main.go:104\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"] [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh + ls /tmp/tidb_cdc_test/ availability cov.availability.26352637.out cov.availability.27772779.out cov.availability.29452947.out cov.availability.30553057.out cov.availability.31943196.out cov.availability.35953597.out cov.availability.38263828.out cov.availability.39353937.out cov.availability.40774079.out cov.availability.45774579.out cov.availability.46924694.out cov.availability.52745276.out cov.availability.54485450.out cov.availability.55545556.out cov.availability.57735775.out cov.availability.59625964.out cov.availability.cli.2580.out cov.availability.cli.2681.out cov.http_proxies.cli.12246.out cov.http_proxies.cli.12336.out cov.sequence.cli.14731.out http_proxies sequence sql_res.availability.txt sql_res.sequence.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G18.tar.gz /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/tikv2.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/tikv_down.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/http_proxies/test_proxy.log /tmp/tidb_cdc_test/http_proxies/tikv2.log /tmp/tidb_cdc_test/http_proxies/tikv3.log /tmp/tidb_cdc_test/http_proxies/cdc.log /tmp/tidb_cdc_test/http_proxies/tidb.log /tmp/tidb_cdc_test/http_proxies/pd1.log /tmp/tidb_cdc_test/http_proxies/down_pd.log /tmp/tidb_cdc_test/http_proxies/tikv1.log /tmp/tidb_cdc_test/http_proxies/tidb_down.log /tmp/tidb_cdc_test/http_proxies/stdout.log /tmp/tidb_cdc_test/http_proxies/tidb_other.log /tmp/tidb_cdc_test/http_proxies/tikv_down.log /tmp/tidb_cdc_test/http_proxies/tidb-slow.log /tmp/tidb_cdc_test/sequence/sync_diff_inspector.log /tmp/tidb_cdc_test/sequence/tikv3/db/000005.log /tmp/tidb_cdc_test/sequence/tikv2.log /tmp/tidb_cdc_test/sequence/tikv3.log /tmp/tidb_cdc_test/sequence/cdc.log /tmp/tidb_cdc_test/sequence/pd1/hot-region/000001.log /tmp/tidb_cdc_test/sequence/pd1/region-meta/000001.log /tmp/tidb_cdc_test/sequence/tidb.log /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/sequence/pd1.log /tmp/tidb_cdc_test/sequence/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/sequence/tiflash/log/error.log /tmp/tidb_cdc_test/sequence/tiflash/log/proxy.log /tmp/tidb_cdc_test/sequence/tiflash/log/server.log /tmp/tidb_cdc_test/sequence/tikv1/db/000005.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/sequence/down_pd.log /tmp/tidb_cdc_test/sequence/tikv1.log /tmp/tidb_cdc_test/sequence/tidb_down.log /tmp/tidb_cdc_test/sequence/stdout.log /tmp/tidb_cdc_test/sequence/tidb_other.log /tmp/tidb_cdc_test/sequence/tikv2/db/000005.log /tmp/tidb_cdc_test/sequence/tikv_down.log /tmp/tidb_cdc_test/sequence/tidb-slow.log /tmp/tidb_cdc_test/sequence/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/sequence/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/sequence/tikv_down/db/000005.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_stop_processor.log /tmp/tidb_cdc_test/availability/tikv2.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/tikv3.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tidb.log /tmp/tidb_cdc_test/availability/pd1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_gap_between_watch_capture.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_cleanup_stale_tasks.server1.log /tmp/tidb_cdc_test/availability/down_pd.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_owner.server2.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log /tmp/tidb_cdc_test/availability/tikv1.log /tmp/tidb_cdc_test/availability/tidb_down.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_owner_retryable_error.server2.log /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log /tmp/tidb_cdc_test/availability/tidb_other.log /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log /tmp/tidb_cdc_test/availability/stdouttest_hang_up_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_kill_capture.server1.log /tmp/tidb_cdc_test/availability/tikv_down.log /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log /tmp/tidb_cdc_test/availability/tidb-slow.log /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log /tmp/tidb_cdc_test/availability/stdouttest_expire_owner.server1.log /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log /tmp/tidb_cdc_test/availability/stdouttest_kill_owner.server2.log /tmp/tidb_cdc_test/http_proxies/test_proxy.log /tmp/tidb_cdc_test/http_proxies/tikv2.log /tmp/tidb_cdc_test/http_proxies/tikv3.log /tmp/tidb_cdc_test/http_proxies/cdc.log /tmp/tidb_cdc_test/http_proxies/tidb.log /tmp/tidb_cdc_test/http_proxies/pd1.log /tmp/tidb_cdc_test/http_proxies/down_pd.log /tmp/tidb_cdc_test/http_proxies/tikv1.log /tmp/tidb_cdc_test/http_proxies/tidb_down.log /tmp/tidb_cdc_test/http_proxies/stdout.log /tmp/tidb_cdc_test/http_proxies/tidb_other.log /tmp/tidb_cdc_test/http_proxies/tikv_down.log /tmp/tidb_cdc_test/http_proxies/tidb-slow.log /tmp/tidb_cdc_test/sequence/sync_diff_inspector.log /tmp/tidb_cdc_test/sequence/tikv3/db/000005.log /tmp/tidb_cdc_test/sequence/tikv2.log /tmp/tidb_cdc_test/sequence/tikv3.log /tmp/tidb_cdc_test/sequence/cdc.log /tmp/tidb_cdc_test/sequence/pd1/hot-region/000001.log /tmp/tidb_cdc_test/sequence/pd1/region-meta/000001.log /tmp/tidb_cdc_test/sequence/tidb.log /tmp/tidb_cdc_test/sequence/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/sequence/pd1.log /tmp/tidb_cdc_test/sequence/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/sequence/tiflash/log/error.log /tmp/tidb_cdc_test/sequence/tiflash/log/proxy.log /tmp/tidb_cdc_test/sequence/tiflash/log/server.log /tmp/tidb_cdc_test/sequence/tikv1/db/000005.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/sequence/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/sequence/down_pd.log /tmp/tidb_cdc_test/sequence/tikv1.log /tmp/tidb_cdc_test/sequence/tidb_down.log /tmp/tidb_cdc_test/sequence/stdout.log /tmp/tidb_cdc_test/sequence/tidb_other.log /tmp/tidb_cdc_test/sequence/tikv2/db/000005.log /tmp/tidb_cdc_test/sequence/tikv_down.log /tmp/tidb_cdc_test/sequence/tidb-slow.log /tmp/tidb_cdc_test/sequence/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/sequence/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/sequence/tikv_down/db/000005.log + ls -alh log-G18.tar.gz -rw-r--r--. 1 jenkins jenkins 9.6M May 6 15:24 log-G18.tar.gz [Pipeline] archiveArtifacts Archiving artifacts Recording fingerprints [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G18' Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... {"level":"warn","ts":1714980293.748208,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002546540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714980293.7484512,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00234f500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":1714980295.7487707,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00234f500/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' ++ stop_tidb_cluster {"level":"warn","ts":1714980294.021252,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002570a80/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714980296.0216486,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002570a80/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh: line 1: 19700 Terminated sleep 130 script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE