Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xcqct-gt7h2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "da8cc2908a23c66f3b444e508dcdecdb76725ae5" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xcqct" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=960c9815-7fb7-430b-a3fb-4fa8814dcdee BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xcqct GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-d4c91f37a2cd54d8a1ba68aedcc7e1b173b616003bd6c892b8b1a5e40c8eb9f7 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xcqct ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build2767337534=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h bash' debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-2gs1h bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3992 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 97118216d69addd2 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 25 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 02:58:05 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/api/v2/model.go, cdc/api/v2/model_test.go, docs/swagger/docs.go, docs/swagger/swagger.json, docs/swagger/swagger.yaml, pkg/cmd/util/helper_test.go, pkg/config/config_test_data.go, pkg/config/replica_config.go, pkg/config/replica_config_test.go, pkg/config/sink.go, pkg/orchestrator/reactor_state_test.go, pkg/sink/codec/common/config.go, pkg/sink/codec/open/open_protocol_encoder_test.go, pkg/sink/codec/open/open_protocol_message.go, tests/integration_tests/api_v2/cases.go, tests/integration_tests/api_v2/model.go] [Pipeline] echo diff file not matched: cdc/api/v2/model.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-76ad8e6) 203833344 bytes in 1.39 secs (146898832 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/.git/ .git HEAD is now at 76ad8e6db fix tests POST git-upload-pack (686 bytes) POST git-upload-pack (973 bytes) POST git-upload-pack (gzip 1823 to 971 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 8c249ff83..93530e277 refs/pull/10974/head -> origin/pr/10974/head Previous HEAD position was 76ad8e6db fix tests HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) ๐Ÿšง Checkouting to base SHA:fd3a6e756d0d488c7e60bedb05e21e42320c3dc7... HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) 47417d775 mounter(ticdc): calculate row level checksum for timestmap by using UTC time zone (#10564) (#10646) 3ab47635f *: release-7.5 bump tidb dependencies (#10970) ๐Ÿšง Pre-merge heads of pull requests to base SHA: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 ... Updating fd3a6e756..93530e277 Fast-forward cdc/api/v2/model.go | 21 +++++++++- cdc/api/v2/model_test.go | 1 + docs/swagger/docs.go | 23 +++++++++++ docs/swagger/swagger.json | 23 +++++++++++ docs/swagger/swagger.yaml | 15 +++++++ pkg/cmd/util/helper_test.go | 2 + pkg/config/config_test_data.go | 15 +++++-- pkg/config/replica_config.go | 1 + pkg/config/replica_config_test.go | 3 ++ pkg/config/sink.go | 8 ++++ pkg/orchestrator/reactor_state_test.go | 3 ++ pkg/sink/codec/common/config.go | 8 ++++ pkg/sink/codec/open/open_protocol_encoder_test.go | 49 +++++++++++++++++++++++ pkg/sink/codec/open/open_protocol_message.go | 7 +++- tests/integration_tests/api_v2/cases.go | 2 + tests/integration_tests/api_v2/model.go | 32 +++++++++------ 16 files changed, 194 insertions(+), 19 deletions(-) ๐Ÿงพ Pre-merged result: 93530e27720635216b5d5fcf531a48b801612005 93530e277 Merge remote-tracking branch 'upstream/release-7.5' into cherry-pick-10915-to-release-7.5 fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) 47417d775 mounter(ticdc): calculate row level checksum for timestmap by using UTC time zone (#10564) (#10646) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-fd3a6e7-93530e2 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh release-7.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 214 0 --:--:-- --:--:-- --:--:-- 214 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1585 0 --:--:-- --:--:-- --:--:-- 1640 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1165 0 --:--:-- --:--:-- --:--:-- 1171 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 992 0 --:--:-- --:--:-- --:--:-- 1000 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/61c4bd61254323d2dc68a8de048000ea036e67ac/centos7/tidb-server.tar.gz 2024-05-06 10:58:43 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/61c4bd61254323d2dc68a8de048000ea036e67ac/centos7/tidb-server.tar.gz [511672373/511672373] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz 2024-05-06 10:58:59 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz [173223227/173223227] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz 2024-05-06 10:59:45 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz [810269604/810269604] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz 2024-05-06 11:00:05 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz [263167205/263167205] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 11:00:09 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 11:00:11 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 11:00:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 11:00:13 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 11:00:18 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 11:00:34 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 11:00 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 11:00 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 11:00 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 11:00 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 1 10:52 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 1 23:57 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 11:00 . drwxr-sr-x 3 jenkins jenkins 4.0K May 6 11:00 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 11:00 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 11:00 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 1 10:52 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 1 23:57 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K May 6 11:00 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 11:00 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:00:39" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc ./cmd/cdc/main.go go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/gin-gonic/gin v1.8.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go v0.112.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading blainsmith.com/go/seahash v1.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/philhofer/fwd v1.1.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/go-playground/validator/v10 v10.11.1 go: downloading github.com/pelletier/go-toml/v2 v2.0.5 go: downloading github.com/ugorji/go/codec v1.2.7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading golang.org/x/text v0.14.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/go-playground/universal-translator v0.18.0 go: downloading github.com/leodido/go-urn v1.2.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/swaggo/swag v1.8.3 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/go-playground/locales v0.14.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.20.6 go: downloading github.com/go-openapi/jsonreference v0.20.2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/go-openapi/jsonpointer v0.19.6 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/apache/thrift v0.16.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/go-openapi/swag v0.22.3 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:03:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:03:53" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:04:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:04:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:04:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading gorm.io/gorm v1.23.8 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/glebarez/go-sqlite v1.17.3 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/glebarez/sqlite v1.4.6 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.16.8 go: downloading modernc.org/sqlite v1.17.3 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.1.1 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:04:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-26-g93530e277" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 03:04:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=93530e27720635216b5d5fcf531a48b801612005" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-26-g93530e277"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.1G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 11:09 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 11:00 .. -rwxr-xr-x 1 jenkins jenkins 204M May 6 11:09 cdc -rwxr-xr-x 1 jenkins jenkins 339M May 6 11:08 cdc.test -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_storage_consumer + ./bin/cdc version Release Version: v7.5.1-26-g93530e277 Git Commit Hash: 93530e27720635216b5d5fcf531a48b801612005 Git Branch: HEAD UTC Build Time: 2024-05-06 03:04:01 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-fd3a6e7-93530e2) 1137751040 bytes in 41.28 secs (27559411 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.7G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 11:09 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 11:00 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 6 11:09 bin -rwxr-xr-x 1 jenkins jenkins 204M May 6 11:09 cdc -rwxr-xr-x 1 jenkins jenkins 339M May 6 11:08 cdc.test -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 11:04 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K May 6 11:09 etc -rwxr-xr-x 1 jenkins jenkins 17M May 6 11:09 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 11:09 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 11:09 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 6 11:09 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 11:09 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 6 11:09 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 11:09 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 6 11:09 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 11:09 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 11:09 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 11:09 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 88M May 6 11:09 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 6 11:09 minio -rwxr-xr-x 1 jenkins jenkins 33M May 6 11:09 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M May 6 11:09 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M May 6 11:09 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M May 6 11:09 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M May 6 11:09 pd-server -rwxr-xr-x 1 jenkins jenkins 25M May 6 11:09 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M May 6 11:09 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 6 11:09 share -rwxr-xr-x 1 jenkins jenkins 29M May 6 11:09 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M May 6 11:09 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M May 6 11:09 tidb-server -rwxr-xr-x 1 jenkins jenkins 377M May 6 11:09 tiflash -rwxr-xr-x 1 jenkins jenkins 353M May 6 11:09 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 86.87 secs (38777289 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-03w0b-tdz47 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bccfef083b5c3c0e13fa768bf7c78b40624fa8e2" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-03w0b" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xx7kz-trvjc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "65042da8519de1f0f28c97a6c1e86d8338b3fc89" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xx7kz" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p01sh-dgv8l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fe4bd29eec732fcd8c6224cb95d7f7e02499077b" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p01sh" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4c01823c; decorates RemoteLauncher[hudson.remoting.Channel@1fa7a48d:JNLP4-connect connection from 10.233.86.247/10.233.86.247:53032] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@9deef0d; decorates RemoteLauncher[hudson.remoting.Channel@7a24113b:JNLP4-connect connection from 10.233.84.200/10.233.84.200:57446] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@f7f3023; decorates RemoteLauncher[hudson.remoting.Channel@7fff7815:JNLP4-connect connection from 10.233.108.155/10.233.108.155:43540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] node Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-gbd59-qsq3m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "880eae688f994592719709504fc6c41ad63763fb" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-gbd59" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5ce22856; decorates RemoteLauncher[hudson.remoting.Channel@5f7be863:JNLP4-connect connection from 10.233.106.124/10.233.106.124:34824] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-3p4mz-wrxtt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b8daa818e683585defbc85520a47ea1c6be0c441" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-3p4mz" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wfn7n-g73ps Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "53023c20f3f5f337b792e52a432d19245ae813ca" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wfn7n" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-08x25-hb1xm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "dd038de90b780df907a374398c38a1a638312017" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-08x25" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2q72b-fsm6w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "33ffebc909128e4f6c335962cf40a92b9e77185b" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2q72b" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hmlbh-dkw91 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f9212ea0aa9ed4a45beb6c38e971ea90693b74c9" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hmlbh" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hc0zh-hm04c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e23fe92cbfaafeef71a37157decb346a164a7c94" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hc0zh" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 6.54 secs (515125782 bytes/sec) [Pipeline] { [Pipeline] podTemplate Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro lossy_ddl storage_csv_update PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=41f790c8-b75e-4bc4-80bc-14371ce8eaa9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-03w0b GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-03w0b ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] podTemplate Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] { [Pipeline] checkout [Pipeline] node The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 [Pipeline] node [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@61d25719; decorates RemoteLauncher[hudson.remoting.Channel@7bc4fbc8:JNLP4-connect connection from 10.233.100.197/10.233.100.197:45502] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4b36d7c5; decorates RemoteLauncher[hudson.remoting.Channel@13b8d8d6:JNLP4-connect connection from 10.233.126.42/10.233.126.42:49754] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@18521980; decorates RemoteLauncher[hudson.remoting.Channel@5d7c1d42:JNLP4-connect connection from 10.233.84.62/10.233.84.62:47484] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7f5802f1; decorates RemoteLauncher[hudson.remoting.Channel@4d84a4ae:JNLP4-connect connection from 10.233.107.14/10.233.107.14:39286] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@74572fdf; decorates RemoteLauncher[hudson.remoting.Channel@6e1fb4ed:JNLP4-connect connection from 10.233.86.53/10.233.86.53:60378] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1e46d08e; decorates RemoteLauncher[hudson.remoting.Channel@411a7d5:JNLP4-connect connection from 10.233.126.112/10.233.126.112:38228] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wt7x1-dqxs6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d1dba00173e2bd9eee39bca8897bebb5d134542c" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wt7x1" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2g7n2-1mfmh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "590a53607d146efe29a82878fb0cb050d385e9f7" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2g7n2" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-cqs7j-27brg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e9b32d3fcf1bf6848d00f94160d35b3edbf5e533" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-cqs7j" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-8fd2l-0fxsv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2c043490b9d6ce69602043333230b23d6ada99c0" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-8fd2l" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-6skmn-k2cp1 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f7a3d6a5cadd30cc30bdcfc39be408b5fda43305" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-6skmn" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p74d0-c18gp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cb475d14fbb91867be8fcad6c666f0dc323a472d" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p74d0" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-r9mlx-d5nrb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "894fa42fcb916b5c2b21914c905d58bf4bee6878" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-r9mlx" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-s4b7x-21s2v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "83cf5d28e6ccb31cb682995b53017f84ac909b82" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-s4b7x" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p1qbp-kvfxg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f12233c34b8dc724710d38ae4753568552bf16f6" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p1qbp" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-lkg7f-x485z --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6eb2f0866ffcd425095e2f484a41a2077ab42bf5" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-lkg7f" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f8ffd70001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:1139, start at 2024-05-06 11:13:04.397815841 +0800 CST m=+5.074955723 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:04.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:04.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:04.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f8ffd70001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:1139, start at 2024-05-06 11:13:04.397815841 +0800 CST m=+5.074955723 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:04.404 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:04.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:04.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f8fffa80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:1218, start at 2024-05-06 11:13:04.492317488 +0800 CST m=+5.118399006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:04.498 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:04.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:04.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 11:13:07 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.25732575.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:13:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b649-2f94-4b11-8dbe-be140c32e91b {"id":"88b1b649-2f94-4b11-8dbe-be140c32e91b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be3d5baf3 88b1b649-2f94-4b11-8dbe-be140c32e91b /tidb/cdc/default/default/upstream/7365719340012160782 {"id":7365719340012160782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b649-2f94-4b11-8dbe-be140c32e91b {"id":"88b1b649-2f94-4b11-8dbe-be140c32e91b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be3d5baf3 88b1b649-2f94-4b11-8dbe-be140c32e91b /tidb/cdc/default/default/upstream/7365719340012160782 {"id":7365719340012160782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88b1b649-2f94-4b11-8dbe-be140c32e91b {"id":"88b1b649-2f94-4b11-8dbe-be140c32e91b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be3d5baf3 88b1b649-2f94-4b11-8dbe-be140c32e91b /tidb/cdc/default/default/upstream/7365719340012160782 {"id":7365719340012160782,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:13:10 CST 2024] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.26232625.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 17.21 secs (195704080 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:13:13 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f427e430-601e-41bf-9200-5860cec4c949 {"id":"f427e430-601e-41bf-9200-5860cec4c949","address":"127.0.0.1:8400","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4be3d5e61e f427e430-601e-41bf-9200-5860cec4c949 /tidb/cdc/default/default/upstream/7365719345104753558 {"id":7365719345104753558,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f427e430-601e-41bf-9200-5860cec4c949 {"id":"f427e430-601e-41bf-9200-5860cec4c949","address":"127.0.0.1:8400","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4be3d5e61e f427e430-601e-41bf-9200-5860cec4c949 /tidb/cdc/default/default/upstream/7365719345104753558 {"id":7365719345104753558,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f427e430-601e-41bf-9200-5860cec4c949 {"id":"f427e430-601e-41bf-9200-5860cec4c949","address":"127.0.0.1:8400","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35118f4be3d5e61e f427e430-601e-41bf-9200-5860cec4c949 /tidb/cdc/default/default/upstream/7365719345104753558 {"id":7365719345104753558,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table bdr_mode.t1 exists table bdr_mode.t1 exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2691.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365719340012160782,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2024-05-06T11:13:16.048386803+08:00","start_ts":449567836326592513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567836326592513,"checkpoint_ts":449567836326592513,"checkpoint_time":"2024-05-06 11:13:15.948"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.2717.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365719345104753558,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2024-05-06T11:13:18.022427811+08:00","start_ts":449567836835938306,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567836835938306,"checkpoint_ts":449567836835938306,"checkpoint_time":"2024-05-06 11:13:17.891"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 6.20 secs (543539101 bytes/sec) [Pipeline] { [Pipeline] cache table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 does not exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark exists check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:13:26 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 27.26 secs (123557378 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=fad8e0d2-894e-49fb-ad0f-33339db539d1 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xx7kz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-xx7kz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [Pipeline] podTemplate + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=a0ec59fa-16be-4ff7-a405-7b5da5c51978 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-gbd59 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-gbd59 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] podTemplate + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 [Pipeline] { Run cases: http_api http_api_tls api_v2 kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=0f4ac3fd-6d89-450f-9e7b-69c9def386b8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p01sh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p01sh ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] node [Pipeline] node WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl (66kB) [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Collecting certifi>=2017.4.17 (from requests==2.26.0) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f92efd80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:3832, start at 2024-05-06 11:13:52.658621059 +0800 CST m=+5.143535878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:52.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:52.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:52.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f92efd80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:3832, start at 2024-05-06 11:13:52.658621059 +0800 CST m=+5.143535878 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:52.665 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:52.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:52.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f92f14c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:3921, start at 2024-05-06 11:13:52.742447463 +0800 CST m=+5.179115258 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:15:52.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:13:52.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:03:52.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4181568a; decorates RemoteLauncher[hudson.remoting.Channel@4c9413e8:JNLP4-connect connection from 10.233.126.107/10.233.126.107:51424] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7bea2776; decorates RemoteLauncher[hudson.remoting.Channel@17cbb2cc:JNLP4-connect connection from 10.233.84.9/10.233.84.9:43994] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@59e28a2d; decorates RemoteLauncher[hudson.remoting.Channel@15cc1916:JNLP4-connect connection from 10.233.100.91/10.233.100.91:35036] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3fe85301; decorates RemoteLauncher[hudson.remoting.Channel@ba8a45d:JNLP4-connect connection from 10.233.86.242/10.233.86.242:47216] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Downloading https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl (163kB) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@73e2a8b; decorates RemoteLauncher[hudson.remoting.Channel@5834b6dc:JNLP4-connect connection from 10.233.107.155/10.233.107.155:43012] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 40 min No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@73dcd2f5; decorates RemoteLauncher[hudson.remoting.Channel@72fa03bf:JNLP4-connect connection from 10.233.108.15/10.233.108.15:53474] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3f160d97; decorates RemoteLauncher[hudson.remoting.Channel@28fcad28:JNLP4-connect connection from 10.233.107.151/10.233.107.151:50536] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@43fafd0; decorates RemoteLauncher[hudson.remoting.Channel@71d24865:JNLP4-connect connection from 10.233.108.153/10.233.108.153:44760] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@54f4f05a; decorates RemoteLauncher[hudson.remoting.Channel@b8c239:JNLP4-connect connection from 10.233.106.14/10.233.106.14:59520] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@71451d99; decorates RemoteLauncher[hudson.remoting.Channel@7ea40c32:JNLP4-connect connection from 10.233.106.45/10.233.106.45:32892] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/b0/53/aa91e163dcfd1e5b82d8a890ecf13314e3e149c05270cc644581f77f17fd/urllib3-1.26.18-py2.py3-none-any.whl (143kB) Installing collected packages: idna, certifi, urllib3, requests Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 Found existing installation: certifi 2023.5.7 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2024.2.2 idna-3.7 requests-2.26.0 urllib3-1.26.18 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] cache The 1 times to try to start tidb cluster... [Mon May 6 11:13:55 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.51785180.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-n9hfw-pd96k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9382a8cff6e2c74a26706be72439c18208769e45" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-n9hfw" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:13:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:13:59 CST 2024] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.52255227.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Agent ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-4tsns-8np7v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "de4e576a0ff0c079816d6b78f22c2836d78a392f" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-4tsns" name: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Running on ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:02 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/edca1e15-5adb-435c-936f-329cc8025c83 {"id":"edca1e15-5adb-435c-936f-329cc8025c83","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/__cdc_meta__/owner/22318f4be494191b edca1e15-5adb-435c-936f-329cc8025c83 /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/edca1e15-5adb-435c-936f-329cc8025c83 {"id":"edca1e15-5adb-435c-936f-329cc8025c83","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/__cdc_meta__/owner/22318f4be494191b edca1e15-5adb-435c-936f-329cc8025c83 /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55b943f8-a410-44b0-a170-f2abf15d09ac {"id":"55b943f8-a410-44b0-a170-f2abf15d09ac","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/edca1e15-5adb-435c-936f-329cc8025c83 {"id":"edca1e15-5adb-435c-936f-329cc8025c83","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be49418f2 55b943f8-a410-44b0-a170-f2abf15d09ac /tidb/cdc/default/__cdc_meta__/owner/22318f4be494191b edca1e15-5adb-435c-936f-329cc8025c83 /tidb/cdc/default/default/upstream/7365719552579856698 {"id":7365719552579856698,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f938da8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:1219, start at 2024-05-06 11:14:02.747901016 +0800 CST m=+5.173030904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:02.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:02.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:02.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f938da8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:1219, start at 2024-05-06 11:14:02.747901016 +0800 CST m=+5.173030904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:02.755 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:02.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:02.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f938f44000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:1298, start at 2024-05-06 11:14:02.849788339 +0800 CST m=+5.196749811 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:02.858 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:02.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:02.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93a7700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:1175, start at 2024-05-06 11:14:04.428978411 +0800 CST m=+5.258667381 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:04.441 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:04.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:04.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:14:05 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.25532555.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93a7700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:1175, start at 2024-05-06 11:14:04.428978411 +0800 CST m=+5.258667381 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:04.441 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:04.430 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:04.430 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93a9f4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:1247, start at 2024-05-06 11:14:04.554413105 +0800 CST m=+5.314489833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:04.561 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:04.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:04.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table capture_suicide_while_balance_table.t3 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9029ff7-747b-4a60-8c86-be79742a2957 {"id":"f9029ff7-747b-4a60-8c86-be79742a2957","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf42f0 f9029ff7-747b-4a60-8c86-be79742a2957 /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9029ff7-747b-4a60-8c86-be79742a2957 {"id":"f9029ff7-747b-4a60-8c86-be79742a2957","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf42f0 f9029ff7-747b-4a60-8c86-be79742a2957 /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9029ff7-747b-4a60-8c86-be79742a2957 {"id":"f9029ff7-747b-4a60-8c86-be79742a2957","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf42f0 f9029ff7-747b-4a60-8c86-be79742a2957 /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2572.out cli tso query --pd=http://127.0.0.1:2379 ***************** properties ***************** "workload"="core" "recordcount"="50" "operationcount"="0" "threadcount"="10" "readproportion"="0" "scanproportion"="0" "mysql.db"="consistent_replicate_ddl" "updateproportion"="0" "mysql.host"="127.0.0.1" "fieldcount"="3" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 12.201046ms INSERT - Takes(s): 0.0, Count: 50, OPS: 8042.1, Avg(us): 2220, Min(us): 922, Max(us): 5965, 95th(us): 6000, 99th(us): 6000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93f7980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb, pid:1277, start at 2024-05-06 11:14:09.519167588 +0800 CST m=+5.144106653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:09.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:09.510 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:09.510 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93f7980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb, pid:1277, start at 2024-05-06 11:14:09.519167588 +0800 CST m=+5.144106653 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:09.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:09.510 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:09.510 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f93f7480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w42pb, pid:1352, start at 2024-05-06 11:14:09.51530334 +0800 CST m=+5.073747476 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:09.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:09.490 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:09.490 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later + set +x + tso='449567850451435521 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567850451435521 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:14:11 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26012603.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later [Mon May 6 11:14:12 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.26632665.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table capture_suicide_while_balance_table.t4 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59dfad55-3762-41d0-a9b5-f386c5530cc1 {"id":"59dfad55-3762-41d0-a9b5-f386c5530cc1","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4c198f5 59dfad55-3762-41d0-a9b5-f386c5530cc1 /tidb/cdc/default/default/upstream/7365719603747244857 {"id":7365719603747244857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59dfad55-3762-41d0-a9b5-f386c5530cc1 {"id":"59dfad55-3762-41d0-a9b5-f386c5530cc1","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4c198f5 59dfad55-3762-41d0-a9b5-f386c5530cc1 /tidb/cdc/default/default/upstream/7365719603747244857 {"id":7365719603747244857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/59dfad55-3762-41d0-a9b5-f386c5530cc1 {"id":"59dfad55-3762-41d0-a9b5-f386c5530cc1","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4c198f5 59dfad55-3762-41d0-a9b5-f386c5530cc1 /tidb/cdc/default/default/upstream/7365719603747244857 {"id":7365719603747244857,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2650.out cli changefeed create --start-ts=449567850451435521 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: fffb3e73-4aec-4724-97ae-538268349953 Info: {"upstream_id":7365719603747244857,"namespace":"default","id":"fffb3e73-4aec-4724-97ae-538268349953","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:14:14.692836298+08:00","start_ts":449567850451435521,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567850451435521,"checkpoint_ts":449567850451435521,"checkpoint_time":"2024-05-06 11:14:09.830"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 749 2248 --:--:-- --:--:-- --:--:-- 2282 { "status": true, "message": "" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:15 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later "is-owner": true, run task successfully }lease 22318f4be494191b revoked check diff failed 1-th time, retry later owner pid: 2668 owner id 7a863b60-3de7-48ca-8b33-52af6a7b11a3 default [Mon May 6 11:14:18 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27722774.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 21.36 secs (157690693 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 2-th time, retry later table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:21 GMT < Content-Length: 911 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/7a863b60-3de7-48ca-8b33-52af6a7b11a3 {"id":"7a863b60-3de7-48ca-8b33-52af6a7b11a3","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d4f5 7a863b60-3de7-48ca-8b33-52af6a7b11a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "7a863b60-3de7-48ca-8b33-52af6a7b11a3 default" | grep id check diff failed 3-th time, retry later table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later "id": "6c1995ca-5807-4d38-9049-5f9785502ba9", run task successfully capture_id: 6c1995ca-5807-4d38-9049-5f9785502ba9 pass test: check health pass test: get status check diff successfully table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... pass test: create changefeed wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:14:25 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> table row_format.finish_mark not exists for 1-th check, retry later table consistent_replicate_ddl.usertable_bak exists table test.simple1 not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 8.46 secs (397936573 bytes/sec) [Pipeline] { [Pipeline] cache table test.simple1 not exists for 2-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later table test.simple1 exists pass test: list changefeed wait process cdc.test exit for 1-th time... table row_format.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... pass test: get changefeed cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:14:33 CST 2024] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.27962798.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: pause changefeed pass test: update changefeed table row_format.finish_mark not exists for 5-th check, retry later wait process 2668 exit for 1-th time... wait process 2668 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2668) - No such process wait process 2668 exit for 3-th time... process 2668 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 6c1995ca-5807-4d38-9049-5f9785502ba9 -A1 | grep '"is-owner": true' table row_format.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {UpstreamID:7365719594807069298 Namespace:default ID:89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:09.230278661 +0800 CST StartTs:449567850254827526 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fc7200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567850281041927} {CheckpointTs:449567856965976066 MinTableBarrierTs:449567857227857922 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09bacb02-2676-44c5-94f8-3c13080a22de {"id":"09bacb02-2676-44c5-94f8-3c13080a22de","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf43b4 09bacb02-2676-44c5-94f8-3c13080a22de /tidb/cdc/default/default/changefeed/info/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"upstream-id":7365719594807069298,"namespace":"default","changefeed-id":"89b8ac89-5abc-4f5a-8e80-135c98a6c5c6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:09.230278661+08:00","start-ts":449567850254827526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567850281041927} /tidb/cdc/default/default/changefeed/status/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":449567856965976066,"min-table-barrier-ts":449567857227857922,"admin-job-type":0} /tidb/cdc/default/default/task/position/09bacb02-2676-44c5-94f8-3c13080a22de/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {UpstreamID:7365719594807069298 Namespace:default ID:89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:09.230278661 +0800 CST StartTs:449567850254827526 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fc7200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567850281041927} {CheckpointTs:449567856965976066 MinTableBarrierTs:449567857227857922 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09bacb02-2676-44c5-94f8-3c13080a22de {"id":"09bacb02-2676-44c5-94f8-3c13080a22de","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf43b4 09bacb02-2676-44c5-94f8-3c13080a22de /tidb/cdc/default/default/changefeed/info/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"upstream-id":7365719594807069298,"namespace":"default","changefeed-id":"89b8ac89-5abc-4f5a-8e80-135c98a6c5c6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:09.230278661+08:00","start-ts":449567850254827526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567850281041927} /tidb/cdc/default/default/changefeed/status/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":449567856965976066,"min-table-barrier-ts":449567857227857922,"admin-job-type":0} /tidb/cdc/default/default/task/position/09bacb02-2676-44c5-94f8-3c13080a22de/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {UpstreamID:7365719594807069298 Namespace:default ID:89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:09.230278661 +0800 CST StartTs:449567850254827526 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fc7200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567850281041927} {CheckpointTs:449567856965976066 MinTableBarrierTs:449567857227857922 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449567856965976067, checkpointTs: 449567856965976066, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09bacb02-2676-44c5-94f8-3c13080a22de {"id":"09bacb02-2676-44c5-94f8-3c13080a22de","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4bf43b4 09bacb02-2676-44c5-94f8-3c13080a22de /tidb/cdc/default/default/changefeed/info/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"upstream-id":7365719594807069298,"namespace":"default","changefeed-id":"89b8ac89-5abc-4f5a-8e80-135c98a6c5c6","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:09.230278661+08:00","start-ts":449567850254827526,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567850281041927} /tidb/cdc/default/default/changefeed/status/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":449567856965976066,"min-table-barrier-ts":449567857227857922,"admin-job-type":0} /tidb/cdc/default/default/task/position/09bacb02-2676-44c5-94f8-3c13080a22de/89b8ac89-5abc-4f5a-8e80-135c98a6c5c6 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365719594807069298 {"id":7365719594807069298,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x "is-owner": true, run task successfully [Mon May 6 11:14:36 CST 2024] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.30223024.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:14:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:24.888705129 +0800 CST StartTs:449567854378352647 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002b38f30 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854391459847} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 11:14:25.108854558 +0800 CST StartTs:449567854443888648 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00381b4d0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854443888654} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5950540c-efb6-465b-b5a7-eac74348641d {"id":"5950540c-efb6-465b-b5a7-eac74348641d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d599 5950540c-efb6-465b-b5a7-eac74348641d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:24.888705129+08:00","start-ts":449567854378352647,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854391459847} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.007892935+08:00","start-ts":449567854417674246,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567856527933441} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.108854558+08:00","start-ts":449567854443888648,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854443888654} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567856383754246,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:24.888705129 +0800 CST StartTs:449567854378352647 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002b38f30 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854391459847} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 11:14:25.108854558 +0800 CST StartTs:449567854443888648 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00381b4d0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854443888654} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5950540c-efb6-465b-b5a7-eac74348641d {"id":"5950540c-efb6-465b-b5a7-eac74348641d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d599 5950540c-efb6-465b-b5a7-eac74348641d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:24.888705129+08:00","start-ts":449567854378352647,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854391459847} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.007892935+08:00","start-ts":449567854417674246,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567856527933441} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.108854558+08:00","start-ts":449567854443888648,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854443888654} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567856383754246,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test1 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:14:24.888705129 +0800 CST StartTs:449567854378352647 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002b38f30 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854391459847} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing changefeedID: default/changefeed-test3 {UpstreamID:7365719625275211005 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2024-05-06 11:14:25.108854558 +0800 CST StartTs:449567854443888648 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00381b4d0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567854443888654} {CheckpointTs:449567854574960673 MinTableBarrierTs:449567858218762247 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567854574960673, checkpointTs: 449567854574960673, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5950540c-efb6-465b-b5a7-eac74348641d {"id":"5950540c-efb6-465b-b5a7-eac74348641d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/6c1995ca-5807-4d38-9049-5f9785502ba9 {"id":"6c1995ca-5807-4d38-9049-5f9785502ba9","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d521 6c1995ca-5807-4d38-9049-5f9785502ba9 /tidb/cdc/default/__cdc_meta__/owner/22318f4be4d3d599 5950540c-efb6-465b-b5a7-eac74348641d /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:14:24.888705129+08:00","start-ts":449567854378352647,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854391459847} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.007892935+08:00","start-ts":449567854417674246,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"stopped","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567856527933441} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7365719625275211005,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2024-05-06T11:14:25.108854558+08:00","start-ts":449567854443888648,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567854443888654} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567856383754246,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":449567854574960673,"min-table-barrier-ts":449567858218762247,"admin-job-type":0} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5950540c-efb6-465b-b5a7-eac74348641d/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6c1995ca-5807-4d38-9049-5f9785502ba9/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365719625275211005 {"id":7365719625275211005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' table row_format.finish_mark not exists for 8-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 8.93 secs (377069103 bytes/sec) [Pipeline] { [Pipeline] cache "address": "127.0.0.1:8300", run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: list changefeed pass test: get changefeed pass test: resume changefeed table row_format.finish_mark not exists for 9-th check, retry later pass test: pause changefeed pass test: rebalance table start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 pass test: move table Verifying upstream PD is started... table row_format.finish_mark not exists for 10-th check, retry later Verifying downstream PD is started... table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Traceback (most recent call last): File "/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/util/test_case.py", line 396, in func(*sys.argv[2:]) File "/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/util/test_case.py", line 293, in get_processor if (resp.status_code != rq.codes.ok): AttributeError: 'NoneType' object has no attribute 'status_code' Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:14:48 CST 2024] <<<<<< run test case row_format success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 11.62 secs (289939360 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f96bf0c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:6522, start at 2024-05-06 11:14:55.056416448 +0800 CST m=+5.147271914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:55.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:55.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:55.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f96bf0c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:6522, start at 2024-05-06 11:14:55.056416448 +0800 CST m=+5.147271914 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:55.063 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:55.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:55.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f96bfb80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:6598, start at 2024-05-06 11:14:55.131393777 +0800 CST m=+5.173415549 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:16:55.137 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:14:55.136 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:04:55.136 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.7988.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449567863260577793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567863260577793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:15:00 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.80148016.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 {"id":"5c88fadd-b6b1-43a3-bcb8-5da5d86f1625","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be583ca01 5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 /tidb/cdc/default/default/upstream/7365719809875356687 {"id":7365719809875356687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 {"id":"5c88fadd-b6b1-43a3-bcb8-5da5d86f1625","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be583ca01 5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 /tidb/cdc/default/default/upstream/7365719809875356687 {"id":7365719809875356687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 {"id":"5c88fadd-b6b1-43a3-bcb8-5da5d86f1625","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be583ca01 5c88fadd-b6b1-43a3-bcb8-5da5d86f1625 /tidb/cdc/default/default/upstream/7365719809875356687 {"id":7365719809875356687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.8062.out cli changefeed create --start-ts=449567863260577793 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 3b70ec4d-87b2-41de-8d23-4a00c86d6c7d Info: {"upstream_id":7365719809875356687,"namespace":"default","id":"3b70ec4d-87b2-41de-8d23-4a00c86d6c7d","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:15:03.589061288+08:00","start_ts":449567863260577793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567863260577793,"checkpoint_ts":449567863260577793,"checkpoint_time":"2024-05-06 11:14:58.693"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x ***************** properties ***************** "updateproportion"="0" "mysql.user"="root" "threadcount"="2" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "workload"="core" "mysql.port"="4000" "requestdistribution"="uniform" "recordcount"="10" "insertproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "scanproportion"="0" "mysql.db"="testSync" ********************************************** Run finished, takes 9.604127ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1888.4, Avg(us): 1843, Min(us): 1031, Max(us): 4279, 95th(us): 5000, 99th(us): 5000 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f97c6040012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:3696, start at 2024-05-06 11:15:11.898708647 +0800 CST m=+5.241460972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:11.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:11.873 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:11.873 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f97c6040012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:3696, start at 2024-05-06 11:15:11.898708647 +0800 CST m=+5.241460972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:11.906 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:11.873 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:11.873 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f97c6b00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:3784, start at 2024-05-06 11:15:11.920993818 +0800 CST m=+5.189393834 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:11.929 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:11.916 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:11.916 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5166.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449567868144320516 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567868144320516 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:15:18 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.52015203.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 23.88 secs (141064791 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6941713d-1246-44e2-88f6-25f45aabc6f2 {"id":"6941713d-1246-44e2-88f6-25f45aabc6f2","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be5c93cf7 6941713d-1246-44e2-88f6-25f45aabc6f2 /tidb/cdc/default/default/upstream/7365719893000290079 {"id":7365719893000290079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6941713d-1246-44e2-88f6-25f45aabc6f2 {"id":"6941713d-1246-44e2-88f6-25f45aabc6f2","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be5c93cf7 6941713d-1246-44e2-88f6-25f45aabc6f2 /tidb/cdc/default/default/upstream/7365719893000290079 {"id":7365719893000290079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6941713d-1246-44e2-88f6-25f45aabc6f2 {"id":"6941713d-1246-44e2-88f6-25f45aabc6f2","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be5c93cf7 6941713d-1246-44e2-88f6-25f45aabc6f2 /tidb/cdc/default/default/upstream/7365719893000290079 {"id":7365719893000290079,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5257.out cli changefeed create --start-ts=449567868144320516 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: eba29b88-cf7a-4a69-a91d-d735338c5571 Info: {"upstream_id":7365719893000290079,"namespace":"default","id":"eba29b88-cf7a-4a69-a91d-d735338c5571","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:15:22.206655436+08:00","start_ts":449567868144320516,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567868144320516,"checkpoint_ts":449567868144320516,"checkpoint_time":"2024-05-06 11:15:17.323"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table drop_tables.c not exists for 1-th check, retry later table drop_tables.c not exists for 2-th check, retry later table drop_tables.c not exists for 3-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 6.00 secs (561264696 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=dca7f750-2e3c-4c04-8d58-e2dfe7648082 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-3p4mz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-3p4mz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 Run cases: gc_safepoint changefeed_pause_resume cli savepoint synced_status PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=e68830ef-39d0-4fa2-9efa-60fd2cff4a09 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wfn7n GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wfn7n ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=ec30c435-7cb7-4c8e-8188-11858efc3e24 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-08x25 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-08x25 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh table drop_tables.c not exists for 4-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 [Pipeline] sh Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=78f90bf0-50f3-431c-8948-13a807fb1d79 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2q72b GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2q72b ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=ea52bae4-c95f-48c0-ae1b-6a7cc3f04ea3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hmlbh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hmlbh ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=9aadeb8a-6b1e-4727-9d51-fb2224633174 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hc0zh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-hc0zh ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table drop_tables.c not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table drop_tables.c exists check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:15:36 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } Cache not saved (inner-step execution failed) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@18f3c077; decorates RemoteLauncher[hudson.remoting.Channel@280dcd71:JNLP4-connect connection from 10.233.126.231/10.233.126.231:46954] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] // dir Post stage [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] sh > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + ls /tmp/tidb_cdc_test/ cov.http_api.26632665.out http_api sql_res.http_api.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G01.tar.gz /tmp/tidb_cdc_test/http_api/tikv2.log /tmp/tidb_cdc_test/http_api/pd3.log /tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log /tmp/tidb_cdc_test/http_api/tiflash/log/error.log /tmp/tidb_cdc_test/http_api/tiflash/log/server.log /tmp/tidb_cdc_test/http_api/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/tikv_down.log /tmp/tidb_cdc_test/http_api/pd2.log /tmp/tidb_cdc_test/http_api/tikv3/db/000005.log /tmp/tidb_cdc_test/http_api/stdout.log /tmp/tidb_cdc_test/http_api/stdout2.log /tmp/tidb_cdc_test/http_api/tikv1.log /tmp/tidb_cdc_test/http_api/tidb_down.log /tmp/tidb_cdc_test/http_api/pd3/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd3/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tikv2/db/000005.log /tmp/tidb_cdc_test/http_api/pd2/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd2/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tidb_other.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/cdc.log /tmp/tidb_cdc_test/http_api/tidb-slow.log /tmp/tidb_cdc_test/http_api/down_pd.log /tmp/tidb_cdc_test/http_api/tikv1/db/000005.log /tmp/tidb_cdc_test/http_api/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/http_api/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/http_api/cdc2.log /tmp/tidb_cdc_test/http_api/pd1.log /tmp/tidb_cdc_test/http_api/stdout1.log /tmp/tidb_cdc_test/http_api/tikv_down/db/000005.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/pd1/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd1/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tidb.log /tmp/tidb_cdc_test/http_api/cdc1.log /tmp/tidb_cdc_test/http_api/tikv3.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/http_api/tikv2.log /tmp/tidb_cdc_test/http_api/pd3.log /tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log /tmp/tidb_cdc_test/http_api/tiflash/log/error.log /tmp/tidb_cdc_test/http_api/tiflash/log/server.log /tmp/tidb_cdc_test/http_api/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data1/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/tikv_down.log /tmp/tidb_cdc_test/http_api/pd2.log /tmp/tidb_cdc_test/http_api/tikv3/db/000005.log [Pipeline] { [Pipeline] { [Pipeline] { Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { /tmp/tidb_cdc_test/http_api/stdout.log /tmp/tidb_cdc_test/http_api/stdout2.log /tmp/tidb_cdc_test/http_api/tikv1.log /tmp/tidb_cdc_test/http_api/tidb_down.log /tmp/tidb_cdc_test/http_api/pd3/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd3/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tikv2/db/000005.log /tmp/tidb_cdc_test/http_api/pd2/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd2/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tidb_other.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data2/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/cdc.log /tmp/tidb_cdc_test/http_api/tidb-slow.log /tmp/tidb_cdc_test/http_api/down_pd.log /tmp/tidb_cdc_test/http_api/tikv1/db/000005.log /tmp/tidb_cdc_test/http_api/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/http_api/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/http_api/cdc2.log /tmp/tidb_cdc_test/http_api/pd1.log /tmp/tidb_cdc_test/http_api/stdout1.log /tmp/tidb_cdc_test/http_api/tikv_down/db/000005.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/http_api/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/http_api/pd1/region-meta/000001.log /tmp/tidb_cdc_test/http_api/pd1/hot-region/000001.log /tmp/tidb_cdc_test/http_api/tidb.log /tmp/tidb_cdc_test/http_api/cdc1.log /tmp/tidb_cdc_test/http_api/tikv3.log [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@69318c6c; decorates RemoteLauncher[hudson.remoting.Channel@f2c1963:JNLP4-connect connection from 10.233.72.109/10.233.72.109:59586] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] cache Cloning the remote Git repository Using shallow clone with depth 1 + ls -alh log-G01.tar.gz -rw-r--r-- 1 jenkins jenkins 2.9M May 6 11:15 log-G01.tar.gz > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99af580008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:1182, start at 2024-05-06 11:15:43.2001539 +0800 CST m=+5.510398357 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.208 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b18c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:1186, start at 2024-05-06 11:15:43.331145018 +0800 CST m=+5.204316624 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.341 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b18c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:1186, start at 2024-05-06 11:15:43.331145018 +0800 CST m=+5.204316624 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.341 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b3f80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:1264, start at 2024-05-06 11:15:43.492826158 +0800 CST m=+5.314338131 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.500 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.486 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.486 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b8bc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:1205, start at 2024-05-06 11:15:43.839884414 +0800 CST m=+5.205440561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b8bc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:1205, start at 2024-05-06 11:15:43.839884414 +0800 CST m=+5.205440561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99ba400015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:1287, start at 2024-05-06 11:15:43.924035853 +0800 CST m=+5.240579789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.931 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99af580008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:1182, start at 2024-05-06 11:15:43.2001539 +0800 CST m=+5.510398357 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.208 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.190 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.190 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99b2900013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:1243, start at 2024-05-06 11:15:43.432960127 +0800 CST m=+5.647566670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:43.442 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:43.446 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:43.446 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2575.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99d1900012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:1138, start at 2024-05-06 11:15:45.406484622 +0800 CST m=+5.028523394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:45.412 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.380 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99d1900012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:1138, start at 2024-05-06 11:15:45.406484622 +0800 CST m=+5.028523394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:45.412 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.380 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99d4a40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:1209, start at 2024-05-06 11:15:45.583402403 +0800 CST m=+5.140421789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:45.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.577 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.577 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:15:46 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.25662568.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 11:15:48 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.24762478.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449567875866558465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567875866558465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:15:48 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26072609.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99dae40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:1189, start at 2024-05-06 11:15:45.997470398 +0800 CST m=+5.563517531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:46.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99dae40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:1189, start at 2024-05-06 11:15:45.997470398 +0800 CST m=+5.563517531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:46.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f99dad80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:1236, start at 2024-05-06 11:15:46.000378404 +0800 CST m=+5.500115595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:46.009 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:45.974 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:45.974 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2578.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26abb2ba-ba25-425b-9995-8fa7742710df {"id":"26abb2ba-ba25-425b-9995-8fa7742710df","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be6461ff0 26abb2ba-ba25-425b-9995-8fa7742710df /tidb/cdc/default/default/upstream/7365720028653415842 {"id":7365720028653415842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26abb2ba-ba25-425b-9995-8fa7742710df {"id":"26abb2ba-ba25-425b-9995-8fa7742710df","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be6461ff0 26abb2ba-ba25-425b-9995-8fa7742710df /tidb/cdc/default/default/upstream/7365720028653415842 {"id":7365720028653415842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26abb2ba-ba25-425b-9995-8fa7742710df {"id":"26abb2ba-ba25-425b-9995-8fa7742710df","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be6461ff0 26abb2ba-ba25-425b-9995-8fa7742710df /tidb/cdc/default/default/upstream/7365720028653415842 {"id":7365720028653415842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x + tso='449567876456120324 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567876456120324 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9a13940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:1191, start at 2024-05-06 11:15:49.630274202 +0800 CST m=+5.198675094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:49.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:49.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:49.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9a13940016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:1191, start at 2024-05-06 11:15:49.630274202 +0800 CST m=+5.198675094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:49.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:49.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:49.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9a15180011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:1279, start at 2024-05-06 11:15:49.724695548 +0800 CST m=+5.216260238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:49.732 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:49.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:49.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 11:15:50 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.25012503.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d501458d-9eb1-4f18-a305-d4c10a94529e {"id":"d501458d-9eb1-4f18-a305-d4c10a94529e","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64650ee d501458d-9eb1-4f18-a305-d4c10a94529e /tidb/cdc/default/default/upstream/7365720026350398076 {"id":7365720026350398076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d501458d-9eb1-4f18-a305-d4c10a94529e {"id":"d501458d-9eb1-4f18-a305-d4c10a94529e","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64650ee d501458d-9eb1-4f18-a305-d4c10a94529e /tidb/cdc/default/default/upstream/7365720026350398076 {"id":7365720026350398076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d501458d-9eb1-4f18-a305-d4c10a94529e {"id":"d501458d-9eb1-4f18-a305-d4c10a94529e","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64650ee d501458d-9eb1-4f18-a305-d4c10a94529e /tidb/cdc/default/default/upstream/7365720026350398076 {"id":7365720026350398076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "operationcount"="0" "mysql.user"="root" "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" "readproportion"="0" "updateproportion"="0" "threadcount"="2" "mysql.port"="4000" "insertproportion"="0" "mysql.db"="sink_retry" "readallfields"="true" "dotransactions"="false" "recordcount"="10" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 8.037289ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2459.3, Avg(us): 1529, Min(us): 805, Max(us): 3985, 95th(us): 4000, 99th(us): 4000 [Mon May 6 11:15:51 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26402642.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:51 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5018f48-ebbb-4a24-b1c3-1b43419b031c {"id":"d5018f48-ebbb-4a24-b1c3-1b43419b031c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64807f6 d5018f48-ebbb-4a24-b1c3-1b43419b031c /tidb/cdc/default/default/upstream/7365720031146600972 {"id":7365720031146600972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5018f48-ebbb-4a24-b1c3-1b43419b031c {"id":"d5018f48-ebbb-4a24-b1c3-1b43419b031c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64807f6 d5018f48-ebbb-4a24-b1c3-1b43419b031c /tidb/cdc/default/default/upstream/7365720031146600972 {"id":7365720031146600972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d5018f48-ebbb-4a24-b1c3-1b43419b031c {"id":"d5018f48-ebbb-4a24-b1c3-1b43419b031c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64807f6 d5018f48-ebbb-4a24-b1c3-1b43419b031c /tidb/cdc/default/default/upstream/7365720031146600972 {"id":7365720031146600972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2655.out cli changefeed create --start-ts=449567875866558465 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 82944967-6473-4407-ae33-f6cf789d456b Info: {"upstream_id":7365720031146600972,"namespace":"default","id":"82944967-6473-4407-ae33-f6cf789d456b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:15:51.721199336+08:00","start_ts":449567875866558465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567875866558465,"checkpoint_ts":449567875866558465,"checkpoint_time":"2024-05-06 11:15:46.781"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64318194-542b-4d8c-a7e1-97f3c29c20fe {"id":"64318194-542b-4d8c-a7e1-97f3c29c20fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64931f4 64318194-542b-4d8c-a7e1-97f3c29c20fe /tidb/cdc/default/default/upstream/7365720037475597296 {"id":7365720037475597296,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64318194-542b-4d8c-a7e1-97f3c29c20fe {"id":"64318194-542b-4d8c-a7e1-97f3c29c20fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64931f4 64318194-542b-4d8c-a7e1-97f3c29c20fe /tidb/cdc/default/default/upstream/7365720037475597296 {"id":7365720037475597296,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64318194-542b-4d8c-a7e1-97f3c29c20fe {"id":"64318194-542b-4d8c-a7e1-97f3c29c20fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be64931f4 64318194-542b-4d8c-a7e1-97f3c29c20fe /tidb/cdc/default/default/upstream/7365720037475597296 {"id":7365720037475597296,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 10b13048-b24d-4421-839d-c008a28fc8ff Info: {"upstream_id":7365720037475597296,"namespace":"default","id":"10b13048-b24d-4421-839d-c008a28fc8ff","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:15:54.155864079+08:00","start_ts":449567876940562433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567876940562433,"checkpoint_ts":449567876940562433,"checkpoint_time":"2024-05-06 11:15:50.878"} Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:15:54 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:54 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 {"id":"807aebcb-c48b-4fdb-8b1f-e764fb45d2b0","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be649030b 807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 /tidb/cdc/default/default/upstream/7365720038494483413 {"id":7365720038494483413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 {"id":"807aebcb-c48b-4fdb-8b1f-e764fb45d2b0","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be649030b 807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 /tidb/cdc/default/default/upstream/7365720038494483413 {"id":7365720038494483413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 {"id":"807aebcb-c48b-4fdb-8b1f-e764fb45d2b0","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be649030b 807aebcb-c48b-4fdb-8b1f-e764fb45d2b0 /tidb/cdc/default/default/upstream/7365720038494483413 {"id":7365720038494483413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2688.out cli changefeed create --start-ts=449567876456120324 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27152717.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: a129a882-8a8a-47b8-a1ce-b4ea25e840cf Info: {"upstream_id":7365720038494483413,"namespace":"default","id":"a129a882-8a8a-47b8-a1ce-b4ea25e840cf","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:15:54.998083455+08:00","start_ts":449567876456120324,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567876456120324,"checkpoint_ts":449567876456120324,"checkpoint_time":"2024-05-06 11:15:49.030"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365720028653415842 449567877940117507 449567876734255109 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x run task successfully check_changefeed_state http://127.0.0.1:2379 12e4e555-a854-4b55-9221-320298d961eb stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=12e4e555-a854-4b55-9221-320298d961eb + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 12e4e555-a854-4b55-9221-320298d961eb -s + info='{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' + echo '{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' { "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null } ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"stopped",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"stopped",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365720028653415842 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:15:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf62c521-c029-4a47-b005-183a61d19fbd {"id":"bf62c521-c029-4a47-b005-183a61d19fbd","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be65ab063 bf62c521-c029-4a47-b005-183a61d19fbd /tidb/cdc/default/default/upstream/7365720050502558295 {"id":7365720050502558295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf62c521-c029-4a47-b005-183a61d19fbd {"id":"bf62c521-c029-4a47-b005-183a61d19fbd","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be65ab063 bf62c521-c029-4a47-b005-183a61d19fbd /tidb/cdc/default/default/upstream/7365720050502558295 {"id":7365720050502558295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bf62c521-c029-4a47-b005-183a61d19fbd {"id":"bf62c521-c029-4a47-b005-183a61d19fbd","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be65ab063 bf62c521-c029-4a47-b005-183a61d19fbd /tidb/cdc/default/default/upstream/7365720050502558295 {"id":7365720050502558295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 beb514ba-f76b-431b-a366-bf193f965848 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=beb514ba-f76b-431b-a366-bf193f965848 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c beb514ba-f76b-431b-a366-bf193f965848 -s + info='{ "upstream_id": 7365720050502558295, "namespace": "default", "id": "beb514ba-f76b-431b-a366-bf193f965848", "state": "normal", "checkpoint_tso": 449567878835077123, "checkpoint_time": "2024-05-06 11:15:58.105", "error": null }' + echo '{ "upstream_id": 7365720050502558295, "namespace": "default", "id": "beb514ba-f76b-431b-a366-bf193f965848", "state": "normal", "checkpoint_tso": 449567878835077123, "checkpoint_time": "2024-05-06 11:15:58.105", "error": null }' { "upstream_id": 7365720050502558295, "namespace": "default", "id": "beb514ba-f76b-431b-a366-bf193f965848", "state": "normal", "checkpoint_tso": 449567878835077123, "checkpoint_time": "2024-05-06 11:15:58.105", "error": null } ++ echo '{' '"upstream_id":' 7365720050502558295, '"namespace":' '"default",' '"id":' '"beb514ba-f76b-431b-a366-bf193f965848",' '"state":' '"normal",' '"checkpoint_tso":' 449567878835077123, '"checkpoint_time":' '"2024-05-06' '11:15:58.105",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720050502558295, '"namespace":' '"default",' '"id":' '"beb514ba-f76b-431b-a366-bf193f965848",' '"state":' '"normal",' '"checkpoint_tso":' 449567878835077123, '"checkpoint_time":' '"2024-05-06' '11:15:58.105",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 15.21 secs (221505115 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9aa9640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:6336, start at 2024-05-06 11:15:59.239483469 +0800 CST m=+5.158837038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:59.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:59.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:59.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9aa9640015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:6336, start at 2024-05-06 11:15:59.239483469 +0800 CST m=+5.158837038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:59.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:59.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:59.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9aaa180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:6408, start at 2024-05-06 11:15:59.28126561 +0800 CST m=+5.141916508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:17:59.289 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:15:59.288 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:05:59.288 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } run task successfully check_changefeed_state http://127.0.0.1:2379 12e4e555-a854-4b55-9221-320298d961eb normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=12e4e555-a854-4b55-9221-320298d961eb + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 12e4e555-a854-4b55-9221-320298d961eb -s + info='{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "normal", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' + echo '{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "normal", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' { "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "normal", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null } ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"normal",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"normal",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365720028653415842 449567878202261507 449567878202261508 table foreign_key.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 12e4e555-a854-4b55-9221-320298d961eb stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=12e4e555-a854-4b55-9221-320298d961eb + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 12e4e555-a854-4b55-9221-320298d961eb -s + info='{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' + echo '{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null }' { "upstream_id": 7365720028653415842, "namespace": "default", "id": "12e4e555-a854-4b55-9221-320298d961eb", "state": "stopped", "checkpoint_tso": 449567878202261508, "checkpoint_time": "2024-05-06 11:15:55.691", "error": null } ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"stopped",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"12e4e555-a854-4b55-9221-320298d961eb",' '"state":' '"stopped",' '"checkpoint_tso":' 449567878202261508, '"checkpoint_time":' '"2024-05-06' '11:15:55.691",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully ***************** properties ***************** "mysql.port"="4000" "mysql.db"="testSync" "mysql.host"="127.0.0.1" "recordcount"="10" "workload"="core" "insertproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" "readproportion"="0" "scanproportion"="0" "threadcount"="2" "readallfields"="true" "dotransactions"="false" "updateproportion"="0" ********************************************** Run finished, takes 3.948879ms INSERT - Takes(s): 0.0, Count: 10, OPS: 4114.8, Avg(us): 726, Min(us): 493, Max(us): 1485, 95th(us): 2000, 99th(us): 2000 check_changefeed_state http://127.0.0.1:2379 770cb1a3-6866-48d3-9370-e2c84c9a371f normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=770cb1a3-6866-48d3-9370-e2c84c9a371f + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 770cb1a3-6866-48d3-9370-e2c84c9a371f -s + info='{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "770cb1a3-6866-48d3-9370-e2c84c9a371f", "state": "normal", "checkpoint_tso": 449567879958626308, "checkpoint_time": "2024-05-06 11:16:02.391", "error": null }' + echo '{ "upstream_id": 7365720028653415842, "namespace": "default", "id": "770cb1a3-6866-48d3-9370-e2c84c9a371f", "state": "normal", "checkpoint_tso": 449567879958626308, "checkpoint_time": "2024-05-06 11:16:02.391", "error": null }' { "upstream_id": 7365720028653415842, "namespace": "default", "id": "770cb1a3-6866-48d3-9370-e2c84c9a371f", "state": "normal", "checkpoint_tso": 449567879958626308, "checkpoint_time": "2024-05-06 11:16:02.391", "error": null } ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"770cb1a3-6866-48d3-9370-e2c84c9a371f",' '"state":' '"normal",' '"checkpoint_tso":' 449567879958626308, '"checkpoint_time":' '"2024-05-06' '11:16:02.391",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720028653415842, '"namespace":' '"default",' '"id":' '"770cb1a3-6866-48d3-9370-e2c84c9a371f",' '"state":' '"normal",' '"checkpoint_tso":' 449567879958626308, '"checkpoint_time":' '"2024-05-06' '11:16:02.391",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365720028653415842 [Mon May 6 11:16:02 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.76577659.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 11:16:05 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/595f92bf-0d8d-4d52-9f0c-b338395ef86a {"id":"595f92bf-0d8d-4d52-9f0c-b338395ef86a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be68855f2 595f92bf-0d8d-4d52-9f0c-b338395ef86a /tidb/cdc/default/default/upstream/7365720095610540167 {"id":7365720095610540167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/595f92bf-0d8d-4d52-9f0c-b338395ef86a {"id":"595f92bf-0d8d-4d52-9f0c-b338395ef86a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be68855f2 595f92bf-0d8d-4d52-9f0c-b338395ef86a /tidb/cdc/default/default/upstream/7365720095610540167 {"id":7365720095610540167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/595f92bf-0d8d-4d52-9f0c-b338395ef86a {"id":"595f92bf-0d8d-4d52-9f0c-b338395ef86a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be68855f2 595f92bf-0d8d-4d52-9f0c-b338395ef86a /tidb/cdc/default/default/upstream/7365720095610540167 {"id":7365720095610540167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_stop_delay.t not exists for 1-th check, retry later Changefeed remove successfully. ID: 12e4e555-a854-4b55-9221-320298d961eb CheckpointTs: 449567878202261508 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7365720028653415842 449567880823701509 449567878202261508 449567879958626308 table foreign_key.finish_mark not exists for 3-th check, retry later run task successfully Changefeed remove successfully. ID: 770cb1a3-6866-48d3-9370-e2c84c9a371f CheckpointTs: 449567881086107654 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7365720028653415842 run task successfully table processor_stop_delay.t exists check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 4-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:09 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> table test.finish_mark not exists for 2-th check, retry later check diff successfully table test.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later check diff failed 1-th time, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:15 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table foreign_key.finish_mark not exists for 7-th check, retry later check diff failed 3-th time, retry later table foreign_key.finish_mark not exists for 8-th check, retry later check diff failed 4-th time, retry later table foreign_key.finish_mark exists check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:20 CST 2024] <<<<<< run test case foreign_key success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:24 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:25 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.host"="127.0.0.1" "scanproportion"="0" "updateproportion"="0" "dotransactions"="false" "workload"="core" "mysql.db"="sink_retry" "requestdistribution"="uniform" "insertproportion"="0" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "recordcount"="10" "threadcount"="2" "readproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 3.704781ms INSERT - Takes(s): 0.0, Count: 10, OPS: 4066.4, Avg(us): 640, Min(us): 383, Max(us): 1509, 95th(us): 2000, 99th(us): 2000 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 30.10 secs (111895998 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9c9c00000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:4270, start at 2024-05-06 11:16:31.117567749 +0800 CST m=+5.472314875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:31.124 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:31.104 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:31.104 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9c9c00000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:4270, start at 2024-05-06 11:16:31.117567749 +0800 CST m=+5.472314875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:31.124 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:31.104 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:31.104 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9c9d700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:4334, start at 2024-05-06 11:16:31.229258821 +0800 CST m=+5.521016656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:31.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:31.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:31.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9cd7cc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:3992, start at 2024-05-06 11:16:34.933826696 +0800 CST m=+5.258797532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:34.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status [Mon May 6 11:16:36 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) tikv_gc_last_run_time 20240506-11:16:34.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:34.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.55765578.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9cd7cc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:3992, start at 2024-05-06 11:16:34.933826696 +0800 CST m=+5.258797532 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:34.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:34.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:34.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9cd8840014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:4078, start at 2024-05-06 11:16:35.021220815 +0800 CST m=+5.291115747 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:35.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:35.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:35.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78444ad1-5ec7-43e4-9f47-eb72b6675251 {"id":"78444ad1-5ec7-43e4-9f47-eb72b6675251","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be701aaf2 78444ad1-5ec7-43e4-9f47-eb72b6675251 /tidb/cdc/default/default/upstream/7365720231684018442 {"id":7365720231684018442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78444ad1-5ec7-43e4-9f47-eb72b6675251 {"id":"78444ad1-5ec7-43e4-9f47-eb72b6675251","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be701aaf2 78444ad1-5ec7-43e4-9f47-eb72b6675251 /tidb/cdc/default/default/upstream/7365720231684018442 {"id":7365720231684018442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78444ad1-5ec7-43e4-9f47-eb72b6675251 {"id":"78444ad1-5ec7-43e4-9f47-eb72b6675251","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be701aaf2 78444ad1-5ec7-43e4-9f47-eb72b6675251 /tidb/cdc/default/default/upstream/7365720231684018442 {"id":7365720231684018442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 6.35 secs (530162166 bytes/sec) [Pipeline] { [Pipeline] cache table changefeed_pause_resume.t1 not exists for 1-th check, retry later [Mon May 6 11:16:40 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.53205322.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9d03700017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:3573, start at 2024-05-06 11:16:37.742032365 +0800 CST m=+5.225741816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:37.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:37.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:37.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9d03700017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:3573, start at 2024-05-06 11:16:37.742032365 +0800 CST m=+5.225741816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:37.748 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:37.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:37.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9d04e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-83xtw, pid:3638, start at 2024-05-06 11:16:37.83784251 +0800 CST m=+5.249318781 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:37.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:37.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:37.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:16:42 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.49224924.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 {"id":"8d3c87a0-a078-4b5e-bfc4-2237cb5fa280","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be70998f3 8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 /tidb/cdc/default/default/upstream/7365720249646459500 {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 {"id":"8d3c87a0-a078-4b5e-bfc4-2237cb5fa280","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be70998f3 8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 /tidb/cdc/default/default/upstream/7365720249646459500 {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 {"id":"8d3c87a0-a078-4b5e-bfc4-2237cb5fa280","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be70998f3 8d3c87a0-a078-4b5e-bfc4-2237cb5fa280 /tidb/cdc/default/default/upstream/7365720249646459500 {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "threadcount"="4" "mysql.host"="127.0.0.1" "insertproportion"="0" "operationcount"="0" "workload"="core" "requestdistribution"="uniform" "updateproportion"="0" "recordcount"="50" "mysql.port"="4000" "readproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "readallfields"="true" "scanproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 16.178187ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3933.2, Avg(us): 1270, Min(us): 796, Max(us): 3976, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table changefeed_pause_resume.t3 exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35362800-e7d7-4fda-b1ee-56ab4e91b96f {"id":"35362800-e7d7-4fda-b1ee-56ab4e91b96f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189af3 35362800-e7d7-4fda-b1ee-56ab4e91b96f /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35362800-e7d7-4fda-b1ee-56ab4e91b96f {"id":"35362800-e7d7-4fda-b1ee-56ab4e91b96f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189af3 35362800-e7d7-4fda-b1ee-56ab4e91b96f /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35362800-e7d7-4fda-b1ee-56ab4e91b96f {"id":"35362800-e7d7-4fda-b1ee-56ab4e91b96f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189af3 35362800-e7d7-4fda-b1ee-56ab4e91b96f /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.4974.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365720262474145725,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2024-05-06T11:16:46.367124959+08:00","start_ts":449567891449184263,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567891449184263,"checkpoint_ts":449567891449184263,"checkpoint_time":"2024-05-06 11:16:46.224"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da22c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:9033, start at 2024-05-06 11:16:47.927742047 +0800 CST m=+5.260427817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:47.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:47.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:47.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da4240010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:3695, start at 2024-05-06 11:16:48.030541633 +0800 CST m=+5.340449595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:48.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:48.009 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:48.009 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff successfully wait process 5325 exit for 1-th time... wait process 5325 exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da22c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:9033, start at 2024-05-06 11:16:47.927742047 +0800 CST m=+5.260427817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:47.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:47.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:47.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da4640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xbr34, pid:9111, start at 2024-05-06 11:16:48.067852803 +0800 CST m=+5.330868321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:48.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:48.075 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:48.075 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da4240010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:3695, start at 2024-05-06 11:16:48.030541633 +0800 CST m=+5.340449595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:48.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:48.009 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:48.009 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9da40c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ht3rj, pid:3773, start at 2024-05-06 11:16:48.0465149 +0800 CST m=+5.298734621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:48.054 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:48.053 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:48.053 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5325) - No such process wait process 5325 exit for 3-th time... process 5325 already exit check_no_capture http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 11:16:51 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.55935595.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 10.73 secs (313917265 bytes/sec) [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10423.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5099.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] cache check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/fe6e11dc-b12f-4877-8a47-e012c68196e4 {UpstreamID:7365720249646459500 Namespace:default ID:fe6e11dc-b12f-4877-8a47-e012c68196e4 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 11:16:43.316304548 +0800 CST StartTs:449567890651742212 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038cf0e0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567890677694470} {CheckpointTs:449567892106379269 MinTableBarrierTs:449567893417099269 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567892106379269, checkpointTs: 449567892106379269, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7fc030a-5b4c-4488-8d4f-da418c2bea85 {"id":"a7fc030a-5b4c-4488-8d4f-da418c2bea85","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7099970 a7fc030a-5b4c-4488-8d4f-da418c2bea85 /tidb/cdc/default/default/changefeed/info/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"upstream-id":7365720249646459500,"namespace":"default","changefeed-id":"fe6e11dc-b12f-4877-8a47-e012c68196e4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:16:43.316304548+08:00","start-ts":449567890651742212,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567890677694470} /tidb/cdc/default/default/changefeed/status/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":449567892106379269,"min-table-barrier-ts":449567893417099269,"admin-job-type":0} /tidb/cdc/default/default/task/position/a7fc030a-5b4c-4488-8d4f-da418c2bea85/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720249646459500 {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/fe6e11dc-b12f-4877-8a47-e012c68196e4 {UpstreamID:7365720249646459500 Namespace:default ID:fe6e11dc-b12f-4877-8a47-e012c68196e4 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 11:16:43.316304548 +0800 CST StartTs:449567890651742212 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038cf0e0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567890677694470} {CheckpointTs:449567892106379269 MinTableBarrierTs:449567893417099269 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567892106379269, checkpointTs: 449567892106379269, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7fc030a-5b4c-4488-8d4f-da418c2bea85 {"id":"a7fc030a-5b4c-4488-8d4f-da418c2bea85","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7099970 a7fc030a-5b4c-4488-8d4f-da418c2bea85 /tidb/cdc/default/default/changefeed/info/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"upstream-id":7365720249646459500,"namespace":"default","changefeed-id":"fe6e11dc-b12f-4877-8a47-e012c68196e4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:16:43.316304548+08:00","start-ts":449567890651742212,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567890677694470} /tidb/cdc/default/default/changefeed/status/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":449567892106379269,"min-table-barrier-ts":449567893417099269,"admin-job-type":0} /tidb/cdc/default/default/task/position/a7fc030a-5b4c-4488-8d4f-da418c2bea85/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720249646459500 {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/fe6e11dc-b12f-4877-8a47-e012c68196e4 {UpstreamID:7365720249646459500 Namespace:default ID:fe6e11dc-b12f-4877-8a47-e012c68196e4 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 11:16:43.316304548 +0800 CST StartTs:449567890651742212 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0038cf0e0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567890677694470} {CheckpointTs:449567892106379269 MinTableBarrierTs:449567893417099269 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567892106379269, checkpointTs: 449567892106379269, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a7fc030a-5b4c-4488-8d4f-da418c2bea85 {"id":"a7fc030a-5b4c-4488-8d4f-da418c2bea85","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7099970 a7fc030a-5b4c-4488-8d4f-da418c2bea85 /tidb/cdc/default/default/changefeed/info/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"upstream-id":7365720249646459500,"namespace":"default","changefeed-id":"fe6e11dc-b12f-4877-8a47-e012c68196e4","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:16:43.316304548+08:00","start-ts":449567890651742212,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567890677694470} /tidb/cdc/default/default/changefeed/status/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":449567892106379269,"min-table-barrier-ts":449567893417099269,"admin-job-type":0} /tidb/cdc/default/default/task/position/a7fc030a-5b4c-4488-8d4f-da418c2bea85/fe6e11dc-b12f-4877-8a47-e012c68196e4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720249646459500 + grep -q 'etcd info' {"id":7365720249646459500,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + set +x + tso='449567893299920897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567893299920897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:16:54 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1046510467.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9ddf580013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:3940, start at 2024-05-06 11:16:51.833730031 +0800 CST m=+5.328404399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:51.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:51.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:51.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9ddf580013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:3940, start at 2024-05-06 11:16:51.833730031 +0800 CST m=+5.328404399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:51.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:51.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:51.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9de0dc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-7h5hw, pid:3999, start at 2024-05-06 11:16:51.920377634 +0800 CST m=+5.362388152 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:18:51.927 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:16:51.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:06:51.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449567893306474497 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567893306474497 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:16:54 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.51475149.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "id": "a7fc030a-5b4c-4488-8d4f-da418c2bea85", "cluster-id": "default" run task successfully check diff successfully capture_id: a7fc030a-5b4c-4488-8d4f-da418c2bea85 check_processor_table_count http://127.0.0.1:2379 fe6e11dc-b12f-4877-8a47-e012c68196e4 a7fc030a-5b4c-4488-8d4f-da418c2bea85 1 run task successfully [Mon May 6 11:16:56 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.52365238.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_processor_table_count http://127.0.0.1:2379 fe6e11dc-b12f-4877-8a47-e012c68196e4 a7fc030a-5b4c-4488-8d4f-da418c2bea85 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449567859036651549" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 11:16:45.295 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 11:16:45.296 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 11:16:45.296 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449567857385144332] [resolvedTs=449567859036651549] [2024/05/06 11:16:45.297 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:16:45.297 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 11:16:45.297 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.662065ms] [2024/05/06 11:16:45.297 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.741559ms] [2024/05/06 11:16:45.302 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:16:45.305 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 11:16:45.305 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 11:16:45.306 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:16:45.306 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 11:16:45.310 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:16:45.312 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 11:16:45.312 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567857385144327,\"CommitTs\":449567857385144332,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":106,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:45.313 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567857385144327,\"CommitTs\":449567857385144332,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":106,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:45.526 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [duration=213.302921ms] [namespace=] [changefeed=] [2024/05/06 11:16:45.526 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567857503109138,\"CommitTs\":449567857503109142,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:45.526 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567857503109138,\"CommitTs\":449567857503109142,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:45.670 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [duration=143.437889ms] [namespace=] [changefeed=] [2024/05/06 11:16:45.670 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567857647288329,\"CommitTs\":449567857647288333,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1_1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:45.871 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567857647288329,\"CommitTs\":449567857647288333,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable1_1\",\"tbl-id\":112,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:46.017 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [duration=145.943591ms] [namespace=] [changefeed=] [2024/05/06 11:16:46.017 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567857778360342,\"CommitTs\":449567857778360346,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:46.218 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567857778360342,\"CommitTs\":449567857778360346,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:46.302 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE DATABASE consistent_replicate_ddl1"] [duration=83.703129ms] [namespace=] [changefeed=] [2024/05/06 11:16:46.302 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567857909432347,\"CommitTs\":449567857922539521,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:46.302 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567857909432347,\"CommitTs\":449567857922539521,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:46.377 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=75.444393ms] [namespace=] [changefeed=] [2024/05/06 11:16:46.377 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858053611549,\"CommitTs\":449567858053611553,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:46.377 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858053611549,\"CommitTs\":449567858053611553,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:46.463 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [duration=85.813395ms] [namespace=] [changefeed=] [2024/05/06 11:16:46.463 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858250219528,\"CommitTs\":449567858250219532,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:46.664 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858250219528,\"CommitTs\":449567858250219532,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:46.938 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [duration=274.068068ms] [namespace=] [changefeed=] [2024/05/06 11:16:46.938 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858381291531,\"CommitTs\":449567858381291535,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:46.938 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858381291531,\"CommitTs\":449567858381291535,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.210 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [duration=271.948437ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.210 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858512363527,\"CommitTs\":449567858512363531,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:47.210 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858512363527,\"CommitTs\":449567858512363531,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl1\",\"tbl-name\":\"\",\"tbl-id\":0,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.415 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="DROP DATABASE consistent_replicate_ddl1"] [duration=204.857824ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.415 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858617221136,\"CommitTs\":449567858617221139,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:47.415 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858617221136,\"CommitTs\":449567858617221139,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.499 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [duration=83.759779ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.499 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858774769670,\"CommitTs\":449567858774769674,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:47.499 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858774769670,\"CommitTs\":449567858774769674,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.647 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [duration=147.487263ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.647 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858774769670,\"CommitTs\":449567858774769674,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2_1\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:47.647 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858774769670,\"CommitTs\":449567858774769674,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable2_1\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.795 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [duration=148.529877ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.795 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567858892472341,\"CommitTs\":449567858892472345,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:47.795 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567858892472341,\"CommitTs\":449567858892472345,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"usertable3_1\",\"tbl-id\":110,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:47.875 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [duration=79.834595ms] [namespace=] [changefeed=] [2024/05/06 11:16:47.875 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567859036651543,\"CommitTs\":449567859036651549,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"check1\",\"tbl-id\":127,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:16:48.076 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567859036651543,\"CommitTs\":449567859036651549,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_ddl\",\"tbl-name\":\"check1\",\"tbl-id\":127,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:16:48.162 +08:00] [INFO] [mysql_ddl_sink.go:221] ["Exec DDL succeeded"] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [duration=85.37986ms] [namespace=] [changefeed=] [2024/05/06 11:16:48.162 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.162 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.262 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.262 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.363 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.363 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}] [checkpointTs=449567859036651549] [2024/05/06 11:16:48.464 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=449567859036651549] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 11:16:48.464 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Mon May 6 11:16:48 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:57 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e94e72ed-6dad-4830-8d68-5c221a6158e8 {"id":"e94e72ed-6dad-4830-8d68-5c221a6158e8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c5cf6 e94e72ed-6dad-4830-8d68-5c221a6158e8 /tidb/cdc/default/default/upstream/7365720306481326969 {"id":7365720306481326969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e94e72ed-6dad-4830-8d68-5c221a6158e8 {"id":"e94e72ed-6dad-4830-8d68-5c221a6158e8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c5cf6 e94e72ed-6dad-4830-8d68-5c221a6158e8 /tidb/cdc/default/default/upstream/7365720306481326969 {"id":7365720306481326969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e94e72ed-6dad-4830-8d68-5c221a6158e8 {"id":"e94e72ed-6dad-4830-8d68-5c221a6158e8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c5cf6 e94e72ed-6dad-4830-8d68-5c221a6158e8 /tidb/cdc/default/default/upstream/7365720306481326969 {"id":7365720306481326969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10513.out cli changefeed create --start-ts=449567893299920897 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Create changefeed successfully! ID: 0d35826b-d725-473b-b1ad-9afc5fb23a6d Info: {"upstream_id":7365720306481326969,"namespace":"default","id":"0d35826b-d725-473b-b1ad-9afc5fb23a6d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:16:58.205879432+08:00","start_ts":449567893299920897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567893299920897,"checkpoint_ts":449567893299920897,"checkpoint_time":"2024-05-06 11:16:53.284"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8fdb755-49ff-4f14-87ee-91e0bc19af76 {"id":"f8fdb755-49ff-4f14-87ee-91e0bc19af76","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c490d f8fdb755-49ff-4f14-87ee-91e0bc19af76 /tidb/cdc/default/default/upstream/7365720295003685556 {"id":7365720295003685556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8fdb755-49ff-4f14-87ee-91e0bc19af76 {"id":"f8fdb755-49ff-4f14-87ee-91e0bc19af76","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c490d f8fdb755-49ff-4f14-87ee-91e0bc19af76 /tidb/cdc/default/default/upstream/7365720295003685556 {"id":7365720295003685556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8fdb755-49ff-4f14-87ee-91e0bc19af76 {"id":"f8fdb755-49ff-4f14-87ee-91e0bc19af76","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be73c490d f8fdb755-49ff-4f14-87ee-91e0bc19af76 /tidb/cdc/default/default/upstream/7365720295003685556 {"id":7365720295003685556,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5203.out cli changefeed create --start-ts=449567893306474497 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 6a54d294-ed95-4295-a63c-87092532fb41 Info: {"upstream_id":7365720295003685556,"namespace":"default","id":"6a54d294-ed95-4295-a63c-87092532fb41","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:16:58.460978595+08:00","start_ts":449567893306474497,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567893306474497,"checkpoint_ts":449567893306474497,"checkpoint_time":"2024-05-06 11:16:53.309"} PASS ***************** properties ***************** "workload"="core" "mysql.db"="testSync" "operationcount"="0" "readproportion"="0" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "readallfields"="true" "insertproportion"="0" "threadcount"="2" "scanproportion"="0" "mysql.port"="4000" "recordcount"="10" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.073336ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3852.9, Avg(us): 740, Min(us): 451, Max(us): 1639, 95th(us): 2000, 99th(us): 2000 coverage: 2.4% of statements in github.com/pingcap/tiflow/... You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:16:59 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2637aab2-9322-45d1-93ab-5bf3f1140ac7 {"id":"2637aab2-9322-45d1-93ab-5bf3f1140ac7","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be74b48f1 2637aab2-9322-45d1-93ab-5bf3f1140ac7 /tidb/cdc/default/default/upstream/7365720320117553808 {"id":7365720320117553808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2637aab2-9322-45d1-93ab-5bf3f1140ac7 {"id":"2637aab2-9322-45d1-93ab-5bf3f1140ac7","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be74b48f1 2637aab2-9322-45d1-93ab-5bf3f1140ac7 /tidb/cdc/default/default/upstream/7365720320117553808 {"id":7365720320117553808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2637aab2-9322-45d1-93ab-5bf3f1140ac7 {"id":"2637aab2-9322-45d1-93ab-5bf3f1140ac7","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be74b48f1 2637aab2-9322-45d1-93ab-5bf3f1140ac7 /tidb/cdc/default/default/upstream/7365720320117553808 {"id":7365720320117553808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5276.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCC7F692B79F69 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 03:16:59 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... check_processor_table_count http://127.0.0.1:2379 fe6e11dc-b12f-4877-8a47-e012c68196e4 a7fc030a-5b4c-4488-8d4f-da418c2bea85 0 run task successfully ***************** properties ***************** "dotransactions"="false" "recordcount"="50" "operationcount"="0" "readproportion"="0" "updateproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" "readallfields"="true" "mysql.port"="4000" "mysql.user"="root" "threadcount"="4" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 20.793144ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3070.7, Avg(us): 1615, Min(us): 1045, Max(us): 5280, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Create changefeed successfully! ID: eb21752a-6e18-4ab2-a8bb-e3424e812a5c Info: {"upstream_id":7365720320117553808,"namespace":"default","id":"eb21752a-6e18-4ab2-a8bb-e3424e812a5c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:17:00.36670008+08:00","start_ts":449567895125491723,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567895125491723,"checkpoint_ts":449567895125491723,"checkpoint_time":"2024-05-06 11:17:00.248"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:16:59 CST 2024] <<<<<< run test case sink_retry success! >>>>>> + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 6.89 secs (488762836 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 3-th time, retry later wait process 4927 exit for 1-th time... wait process 4927 exit for 2-th time... wait process 4927 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (4927) - No such process wait process 4927 exit for 4-th time... process 4927 already exit [Mon May 6 11:17:02 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50385040.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later go: downloading golang.org/x/text v0.14.0 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 4-th time, retry later check diff successfully go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 wait process cdc.test exit for 1-th time... go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > wait process cdc.test exit for 2-th time... go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892274937878,"min-table-barrier-ts":449567892274937878,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892274937878,"min-table-barrier-ts":449567892274937878,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892274937878,"min-table-barrier-ts":449567892274937878,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:17:05 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50965098.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:17:06 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892406272028,"min-table-barrier-ts":449567892406272028,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892406272028,"min-table-barrier-ts":449567892406272028,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365720262474145725 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2024-05-06 11:16:46.367124959 +0800 CST StartTs:449567891449184263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b79170 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567891475398662} {CheckpointTs:449567892274937878 MinTableBarrierTs:449567892274937878 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92dd1f09-cabb-4131-a9c1-f154c5081703 {"id":"92dd1f09-cabb-4131-a9c1-f154c5081703","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7189cc1 92dd1f09-cabb-4131-a9c1-f154c5081703 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365720262474145725,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2024-05-06T11:16:46.367124959+08:00","start-ts":449567891449184263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567891475398662} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449567892406272028,"min-table-barrier-ts":449567892406272028,"admin-job-type":0} /tidb/cdc/default/default/task/position/92dd1f09-cabb-4131-a9c1-f154c5081703/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720262474145725 {"id":7365720262474145725,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 7.49 secs (449562611 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9f1e6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:4126, start at 2024-05-06 11:17:12.269247666 +0800 CST m=+5.562039507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:12.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:12.269 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:12.269 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9f1e6c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:4126, start at 2024-05-06 11:17:12.269247666 +0800 CST m=+5.562039507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:12.278 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:12.269 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:12.269 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9f1f2c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:4192, start at 2024-05-06 11:17:12.314570568 +0800 CST m=+5.544570796 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:12.326 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:12.317 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:12.317 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 4-th check, retry later check diff failed 4-th time, retry later [Mon May 6 11:17:17 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.54245426.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5237cdc3-62cd-49ce-afa0-859efcb8f2ee {"id":"5237cdc3-62cd-49ce-afa0-859efcb8f2ee","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fdf3 5237cdc3-62cd-49ce-afa0-859efcb8f2ee /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5237cdc3-62cd-49ce-afa0-859efcb8f2ee {"id":"5237cdc3-62cd-49ce-afa0-859efcb8f2ee","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fdf3 5237cdc3-62cd-49ce-afa0-859efcb8f2ee /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5237cdc3-62cd-49ce-afa0-859efcb8f2ee {"id":"5237cdc3-62cd-49ce-afa0-859efcb8f2ee","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fdf3 5237cdc3-62cd-49ce-afa0-859efcb8f2ee /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully table partition_table.t exists table partition_table.t1 exists table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 7-th check, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9fc5200013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:3783, start at 2024-05-06 11:17:22.922023942 +0800 CST m=+5.388255296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:22.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:22.939 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:22.939 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 8-th check, retry later check diff failed 2-th time, retry later table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9fc5200013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:3783, start at 2024-05-06 11:17:22.922023942 +0800 CST m=+5.388255296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:22.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:22.939 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:22.939 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2f9fc6ac0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:3878, start at 2024-05-06 11:17:22.995132934 +0800 CST m=+5.371136636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:23.002 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:22.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:22.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table consistent_replicate_gbk.check1 exists check diff successfully table ddl_manager.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later table partition_table.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa00618001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:6883, start at 2024-05-06 11:17:27.093577076 +0800 CST m=+5.208790764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:27.101 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:27.097 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:27.097 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 11:17:28 CST 2024] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.55935595.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5194.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 10-th check, retry later check diff failed 4-th time, retry later table partition_table.finish_mark not exists for 6-th check, retry later + set +x + tso='449567902463164417 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567902463164417 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "requestdistribution"="uniform" "recordcount"="20" "workload"="core" "scanproportion"="0" "operationcount"="0" "readproportion"="0" "updateproportion"="0" "insertproportion"="0" "mysql.user"="root" "mysql.port"="4000" "threadcount"="4" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="changefeed_error" "readallfields"="true" ********************************************** Run finished, takes 11.908453ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3303.9, Avg(us): 2265, Min(us): 1122, Max(us): 5845, 95th(us): 6000, 99th(us): 6000 [Mon May 6 11:17:29 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.52545256.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa00618001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:6883, start at 2024-05-06 11:17:27.093577076 +0800 CST m=+5.208790764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:27.101 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:27.097 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:27.097 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa006c80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j3rc7, pid:6963, start at 2024-05-06 11:17:27.095216622 +0800 CST m=+5.158213719 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:19:27.102 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:17:27.090 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:07:27.090 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 15.54 secs (216775863 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 11-th check, retry later check diff successfully table partition_table.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {UpstreamID:7365720410326131269 Namespace:default ID:c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:17:20.395253804 +0800 CST StartTs:449567900373876741 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a3320 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567900400091140} {CheckpointTs:449567900714663948 MinTableBarrierTs:449567903126388742 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e90107c6-6a0b-4173-9f3b-9799b972a2a4 {"id":"e90107c6-6a0b-4173-9f3b-9799b972a2a4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fe73 e90107c6-6a0b-4173-9f3b-9799b972a2a4 /tidb/cdc/default/default/changefeed/info/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"upstream-id":7365720410326131269,"namespace":"default","changefeed-id":"c81fe5c9-fb19-4e67-b21f-3f1c854a9a82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:17:20.395253804+08:00","start-ts":449567900373876741,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567900400091140} /tidb/cdc/default/default/changefeed/status/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":449567900714663948,"min-table-barrier-ts":449567903126388742,"admin-job-type":0} /tidb/cdc/default/default/task/position/e90107c6-6a0b-4173-9f3b-9799b972a2a4/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {UpstreamID:7365720410326131269 Namespace:default ID:c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:17:20.395253804 +0800 CST StartTs:449567900373876741 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a3320 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567900400091140} {CheckpointTs:449567900714663948 MinTableBarrierTs:449567903126388742 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e90107c6-6a0b-4173-9f3b-9799b972a2a4 {"id":"e90107c6-6a0b-4173-9f3b-9799b972a2a4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fe73 e90107c6-6a0b-4173-9f3b-9799b972a2a4 /tidb/cdc/default/default/changefeed/info/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"upstream-id":7365720410326131269,"namespace":"default","changefeed-id":"c81fe5c9-fb19-4e67-b21f-3f1c854a9a82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:17:20.395253804+08:00","start-ts":449567900373876741,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567900400091140} /tidb/cdc/default/default/changefeed/status/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":449567900714663948,"min-table-barrier-ts":449567903126388742,"admin-job-type":0} /tidb/cdc/default/default/task/position/e90107c6-6a0b-4173-9f3b-9799b972a2a4/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":n+ grep -q 'failed to get info:' ull}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {UpstreamID:7365720410326131269 Namespace:default ID:c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:17:20.395253804 +0800 CST StartTs:449567900373876741 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030a3320 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567900400091140} {CheckpointTs:449567900714663948 MinTableBarrierTs:449567903126388742 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567900714663948, checkpointTs: 449567900714663948, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e90107c6-6a0b-4173-9f3b-9799b972a2a4 {"id":"e90107c6-6a0b-4173-9f3b-9799b972a2a4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7a5fe73 e90107c6-6a0b-4173-9f3b-9799b972a2a4 /tidb/cdc/default/default/changefeed/info/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"upstream-id":7365720410326131269,"namespace":"default","changefeed-id":"c81fe5c9-fb19-4e67-b21f-3f1c854a9a82","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:17:20.395253804+08:00","start-ts":449567900373876741,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567900400091140} /tidb/cdc/default/default/changefeed/status/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":449567900714663948,"min-table-barrier-ts":449567903126388742,"admin-job-type":0} /tidb/cdc/default/default/task/position/e90107c6-6a0b-4173-9f3b-9799b972a2a4/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720410326131269 {"id":7365720410326131269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":n+ grep -q 'etcd info' ull}' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8227.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 12-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 {"id":"e066506b-b53d-4fee-b0d8-3604101f4802","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67d0a e066506b-b53d-4fee-b0d8-3604101f4802 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 {"id":"e066506b-b53d-4fee-b0d8-3604101f4802","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67d0a e066506b-b53d-4fee-b0d8-3604101f4802 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 {"id":"e066506b-b53d-4fee-b0d8-3604101f4802","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67d0a e066506b-b53d-4fee-b0d8-3604101f4802 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5306.out cli changefeed create --start-ts=449567902463164417 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365720455559486807,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:17:33.432989517+08:00","start_ts":449567902463164417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567902463164417,"checkpoint_ts":449567902463164417,"checkpoint_time":"2024-05-06 11:17:28.239"} PASS check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 8-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 13-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449567902463164417, "checkpoint_time": "2024-05-06 11:17:28.239", "error": { "time": "2024-05-06T11:17:33.443944285+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at 449567902463164417" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449567902463164417, "checkpoint_time": "2024-05-06 11:17:28.239", "error": { "time": "2024-05-06T11:17:33.443944285+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at 449567902463164417" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449567902463164417, "checkpoint_time": "2024-05-06 11:17:28.239", "error": { "time": "2024-05-06T11:17:33.443944285+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at 449567902463164417" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449567902463164417, '"checkpoint_time":' '"2024-05-06' '11:17:28.239",' '"error":' '{' '"time":' '"2024-05-06T11:17:33.443944285+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at '449567902463164417"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449567902463164417, '"checkpoint_time":' '"2024-05-06' '11:17:28.239",' '"error":' '{' '"time":' '"2024-05-06T11:17:33.443944285+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at '449567902463164417"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at 449567902463164417' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449567902463164117 is earlier than or equal to GC safepoint at 449567902463164417 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5378.out cli changefeed resume -c changefeed-error check diff failed 2-th time, retry later table partition_table.finish_mark not exists for 9-th check, retry later + set +x + tso='449567903579111425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567903579111425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.db"="multi_capture_1" "insertproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "scanproportion"="0" "updateproportion"="0" "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "workload"="core" "threadcount"="2" ********************************************** Run finished, takes 11.07643ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1736.0, Avg(us): 2133, Min(us): 1084, Max(us): 5321, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "mysql.port"="4000" "threadcount"="2" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.user"="root" "insertproportion"="0" "mysql.db"="multi_capture_2" "scanproportion"="0" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "recordcount"="10" "readproportion"="0" "updateproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 11.785989ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1574.2, Avg(us): 2258, Min(us): 1221, Max(us): 5347, 95th(us): 6000, 99th(us): 6000 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "requestdistribution"="uniform" "mysql.user"="root" "threadcount"="2" "workload"="core" "dotransactions"="false" "recordcount"="10" "scanproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.port"="4000" "readallfields"="true" "insertproportion"="0" "operationcount"="0" "mysql.db"="multi_capture_3" ********************************************** Run finished, takes 10.026025ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1860.8, Avg(us): 1907, Min(us): 1081, Max(us): 4583, 95th(us): 5000, 99th(us): 5000 table ddl_manager.finish_mark not exists for 14-th check, retry later ***************** properties ***************** "readproportion"="0" "workload"="core" "mysql.db"="multi_capture_4" "readallfields"="true" "mysql.host"="127.0.0.1" "dotransactions"="false" "operationcount"="0" "mysql.user"="root" "recordcount"="10" "scanproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "threadcount"="2" "mysql.port"="4000" ********************************************** Run finished, takes 11.530236ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1604.5, Avg(us): 2219, Min(us): 1281, Max(us): 5233, 95th(us): 6000, 99th(us): 6000 [Mon May 6 11:17:36 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83508352.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later + set +x table changefeed_error.usertable not exists for 1-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later check_redo_resolved_ts c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 449567904764788741 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82/meta global resolved ts 449567904437370887 not forward to 449567904764788741 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later table changefeed_error.usertable exists check_redo_resolved_ts c81fe5c9-fb19-4e67-b21f-3f1c854a9a82 449567904764788741 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/c81fe5c9-fb19-4e67-b21f-3f1c854a9a82/meta check diff failed 1-th time, retry later run task successfully check diff failed 4-th time, retry later table partition_table.finish_mark not exists for 11-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:40 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:17:40 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.83938395.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 3-th time... check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2024/05/06 11:17:41.425 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 11:17:41.426 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 11:17:41.429 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 11:17:41.429 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449567903296782362] [resolvedTs=449567904974766082] [2024/05/06 11:17:41.430 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 11:17:41.430 +08:00] [INFO] [s3.go:407] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2024/05/06 11:17:41.430 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:17:41.430 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 11:17:41.434 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.261113ms] [2024/05/06 11:17:41.434 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=4.371119ms] [2024/05/06 11:17:41.436 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:17:41.439 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 11:17:41.439 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 11:17:41.440 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:17:41.440 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] ***************** properties ***************** "requestdistribution"="uniform" "readproportion"="0" "insertproportion"="0" "mysql.db"="changefeed_error" "mysql.user"="root" "recordcount"="20" "updateproportion"="0" "scanproportion"="0" "workload"="core" "mysql.port"="4000" "operationcount"="0" "dotransactions"="false" "readallfields"="true" "threadcount"="4" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.256283ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6816.9, Avg(us): 724, Min(us): 453, Max(us): 1716, 95th(us): 2000, 99th(us): 2000 check diff failed 5-th time, retry later [2024/05/06 11:17:41.445 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:17:41.448 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 11:17:41.448 +08:00] [WARN] [redo.go:283] ["apply DDL"] [ddl="{\"StartTs\":449567903296782353,\"CommitTs\":449567903296782362,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [2024/05/06 11:17:41.449 +08:00] [INFO] [mysql_ddl_sink.go:186] ["Start exec DDL"] [DDL="{\"StartTs\":449567903296782353,\"CommitTs\":449567903296782362,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"db-name\":\"consistent_replicate_gbk\",\"tbl-name\":\"GBKTABLE2\",\"tbl-id\":108,\"is-partition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false}"] [namespace=] [changefeed=] [2024/05/06 11:17:41.450 +08:00] [INFO] [mysql_ddl_sink.go:119] ["Execute DDL failed, but error can be ignored"] [startTs=449567903296782353] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] [2024/05/06 11:17:41.651 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449567904974766082] [2024/05/06 11:17:41.651 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}] [checkpointTs=449567904974766082] table partition_table.finish_mark not exists for 12-th check, retry later [2024/05/06 11:17:41.752 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=449567904974766082] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 11:17:41.752 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] Apply redo log successfully check diff successfully [Mon May 6 11:17:41 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 9.35 secs (360277050 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 17-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:43 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:17:43 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.84428444.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark exists check diff successfully check diff successfully wait process cdc.test exit for 1-th time... check diff successfully {"id":"e066506b-b53d-4fee-b0d8-3604101f4802","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + [[ /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later wait process cdc.test exit for 3-th time... wait process cdc.test exit for 4-th time... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + [[ /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:46 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/ebb0a9ac-91d0-410b-88ed-84a24af117ac {"id":"ebb0a9ac-91d0-410b-88ed-84a24af117ac","address":"127.0.0.1:8303","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5180 ebb0a9ac-91d0-410b-88ed-84a24af117ac /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/ebb0a9ac-91d0-410b-88ed-84a24af117ac {"id":"ebb0a9ac-91d0-410b-88ed-84a24af117ac","address":"127.0.0.1:8303","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5180 ebb0a9ac-91d0-410b-88ed-84a24af117ac /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/67414cf6-c204-4d62-8260-8090bc838744 {"id":"67414cf6-c204-4d62-8260-8090bc838744","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/943c2076-6ed1-450b-9c6e-fba1af7496a3 {"id":"943c2076-6ed1-450b-9c6e-fba1af7496a3","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/ebb0a9ac-91d0-410b-88ed-84a24af117ac {"id":"ebb0a9ac-91d0-410b-88ed-84a24af117ac","address":"127.0.0.1:8303","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5143 67414cf6-c204-4d62-8260-8090bc838744 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df516f 943c2076-6ed1-450b-9c6e-fba1af7496a3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7df5180 ebb0a9ac-91d0-410b-88ed-84a24af117ac /tidb/cdc/default/default/upstream/7365720471105554741 {"id":7365720471105554741,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8489.out cli changefeed create --start-ts=449567903579111425 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Create changefeed successfully! ID: 3f982260-95cc-454f-b18c-47a95968951d Info: {"upstream_id":7365720471105554741,"namespace":"default","id":"3f982260-95cc-454f-b18c-47a95968951d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:17:46.539396148+08:00","start_ts":449567903579111425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567903579111425,"checkpoint_ts":449567903579111425,"checkpoint_time":"2024-05-06 11:17:32.496"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 5-th time... wait process cdc.test exit for 6-th time... check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 7-th time... process cdc.test already exit [Mon May 6 11:17:47 CST 2024] <<<<<< run test case partition_table success! >>>>>> + set +x table multi_capture_1.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later check diff failed 3-th time, retry later table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + [[ /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/e066506b-b53d-4fee-b0d8-3604101f4802 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 7.66 secs (439601336 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 5-th time, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff successfully table multi_capture_4.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 11:17:56 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.56135615.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully ***************** properties ***************** "readallfields"="true" "threadcount"="2" "dotransactions"="false" "mysql.host"="127.0.0.1" "insertproportion"="0" "workload"="core" "scanproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "recordcount"="20" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="multi_capture_1" ********************************************** Run finished, takes 15.115814ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1509.5, Avg(us): 1316, Min(us): 446, Max(us): 5104, 95th(us): 3000, 99th(us): 6000 ***************** properties ***************** "operationcount"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "mysql.port"="4000" "scanproportion"="0" "readallfields"="true" "insertproportion"="0" "readproportion"="0" "mysql.db"="multi_capture_2" "recordcount"="20" "mysql.user"="root" "updateproportion"="0" "workload"="core" "requestdistribution"="uniform" "dotransactions"="false" ********************************************** Run finished, takes 10.526803ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2186.2, Avg(us): 1003, Min(us): 458, Max(us): 1625, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="20" "updateproportion"="0" "readallfields"="true" "dotransactions"="false" "requestdistribution"="uniform" "workload"="core" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.port"="4000" "readproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_3" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 10.974675ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2191.1, Avg(us): 1033, Min(us): 527, Max(us): 1787, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.user"="root" "readproportion"="0" "readallfields"="true" "dotransactions"="false" "mysql.host"="127.0.0.1" "insertproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "updateproportion"="0" "mysql.db"="multi_capture_4" "recordcount"="20" "operationcount"="0" "scanproportion"="0" "workload"="core" "threadcount"="2" ********************************************** Run finished, takes 11.059409ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2003.9, Avg(us): 1047, Min(us): 455, Max(us): 1726, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 25-th check, retry later check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:17:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/255b789f-5271-4d1d-b1cc-4e70061b0103 {"id":"255b789f-5271-4d1d-b1cc-4e70061b0103","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67da0 255b789f-5271-4d1d-b1cc-4e70061b0103 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365720455559486807,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:17:33.432989517+08:00","start-ts":449567902463164417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T11:17:56.95003691+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567909986697219} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449567906237775874,"min-table-barrier-ts":449567906670575622,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/255b789f-5271-4d1d-b1cc-4e70061b0103 {"id":"255b789f-5271-4d1d-b1cc-4e70061b0103","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67da0 255b789f-5271-4d1d-b1cc-4e70061b0103 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365720455559486807,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:17:33.432989517+08:00","start-ts":449567902463164417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T11:17:56.95003691+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567909986697219} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449567906237775874,"min-table-barrier-ts":449567906670575622,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/255b789f-5271-4d1d-b1cc-4e70061b0103 {"id":"255b789f-5271-4d1d-b1cc-4e70061b0103","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67da0 255b789f-5271-4d1d-b1cc-4e70061b0103 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365720455559486807,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T11:17:33.432989517+08:00","start-ts":449567902463164417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-05-06T11:17:56.95003691+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567909986697219} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449567906237775874,"min-table-barrier-ts":449567906670575622,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449567906237775874, "checkpoint_time": "2024-05-06 11:17:42.638", "error": { "time": "2024-05-06T11:17:56.95003691+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449567906237775874, "checkpoint_time": "2024-05-06 11:17:42.638", "error": { "time": "2024-05-06T11:17:56.95003691+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449567906237775874, "checkpoint_time": "2024-05-06 11:17:42.638", "error": { "time": "2024-05-06T11:17:56.95003691+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567906237775874, '"checkpoint_time":' '"2024-05-06' '11:17:42.638",' '"error":' '{' '"time":' '"2024-05-06T11:17:56.95003691+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567906237775874, '"checkpoint_time":' '"2024-05-06' '11:17:42.638",' '"error":' '{' '"time":' '"2024-05-06T11:17:56.95003691+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5706.out cli changefeed remove -c changefeed-error \033[0;36m<<< Run all test success >>>\033[0m table ddl_manager.finish_mark not exists for 26-th check, retry later check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449567906237775874 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:18:02 CST 2024] <<<<<< run test case multi_capture success! >>>>>> table ddl_manager.finish_mark not exists for 27-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 11:18:03 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57865788.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 8.95 secs (376231957 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh table ddl_manager.finish_mark not exists for 28-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: big_txn changefeed_finish force_replicate_table PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=1975ed87-f979-4153-bc91-64667f8651a8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wt7x1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-wt7x1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 4-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=c50e846c-fed7-4b2f-bc8f-6f188d65eefa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2g7n2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-2g7n2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=612f0239-19ef-4419-b373-0c7f01190bb6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-cqs7j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-cqs7j GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 29-th check, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:06 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a678d32c-53d9-47da-9569-d0778e9da44b {"id":"a678d32c-53d9-47da-9569-d0778e9da44b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67dd2 a678d32c-53d9-47da-9569-d0778e9da44b /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a678d32c-53d9-47da-9569-d0778e9da44b {"id":"a678d32c-53d9-47da-9569-d0778e9da44b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67dd2 a678d32c-53d9-47da-9569-d0778e9da44b /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a678d32c-53d9-47da-9569-d0778e9da44b {"id":"a678d32c-53d9-47da-9569-d0778e9da44b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67dd2 a678d32c-53d9-47da-9569-d0778e9da44b /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5834.out cli changefeed create --start-ts=449567902463164417 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365720455559486807,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:18:07.05823776+08:00","start_ts":449567902463164417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567902463164417,"checkpoint_ts":449567902463164417,"checkpoint_time":"2024-05-06 11:17:28.239"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=d04254da-64a2-4d2d-9af2-338f7869906d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-8fd2l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-8fd2l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=ee0e5632-ece7-4223-94c2-5ba499cd22c3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-6skmn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-6skmn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=1bb33039-73ca-463c-aa09-cba77c865f46 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p74d0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p74d0 ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 Run cases: default_value simple cdc_server_tips event_filter [Pipeline] sh + set +x PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=5a1876cc-468c-491f-aabc-4a41f6290793 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-r9mlx GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-r9mlx ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 Run cases: availability http_proxies sequence PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=d903e5b3-2a26-4053-9834-8649a78d3836 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-s4b7x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-s4b7x GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= table ddl_manager.finish_mark not exists for 30-th check, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1765 0 --:--:-- --:--:-- --:--:-- 1769 + info='{"state":"warning","resolved_ts":449567902856118273,"checkpoint_ts":449567902856118273,"last_warning":{"time":"2024-05-06T11:18:08.530988502+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449567902856118273,"checkpoint_ts":449567902856118273,"last_warning":{"time":"2024-05-06T11:18:08.530988502+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449567902856118273,"checkpoint_ts":449567902856118273,"last_warning":{"time":"2024-05-06T11:18:08.530988502+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449567902856118273,"checkpoint_ts":449567902856118273,"last_warning":{"time":"2024-05-06T11:18:08.530988502+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' [Pipeline] sh The 1 times to try to start tidb cluster... + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449567902856118273,"checkpoint_ts":449567902856118273,"last_warning":{"time":"2024-05-06T11:18:08.530988502+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5887.out cli changefeed remove -c changefeed-error-1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=363db027-e81a-4dca-afd4-a790912ecfd7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p1qbp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-p1qbp ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa26dfc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:6744, start at 2024-05-06 11:18:06.471044628 +0800 CST m=+5.809867224 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:06.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:06.463 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:06.463 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa26dfc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:6744, start at 2024-05-06 11:18:06.471044628 +0800 CST m=+5.809867224 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:06.480 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:06.463 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:06.463 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa270f40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:6798, start at 2024-05-06 11:18:06.663837125 +0800 CST m=+5.910960992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:06.672 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:06.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:06.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff successfully Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=9fe53736-ccb7-474d-bd39-2b7882bc9a00 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-lkg7f GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-lkg7f ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449567902856118273 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 31-th check, retry later Verifying downstream PD is started... [Mon May 6 11:18:11 CST 2024] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.80338035.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... The 1 times to try to start tidb cluster... [Pipeline] archiveArtifacts Archiving artifacts wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m Recording fingerprints Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 32-th check, retry later The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d719bee-3d0d-4fac-82ee-49589d63ab89 {"id":"9d719bee-3d0d-4fac-82ee-49589d63ab89","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be86ee5f4 9d719bee-3d0d-4fac-82ee-49589d63ab89 /tidb/cdc/default/default/upstream/7365720638219039435 {"id":7365720638219039435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d719bee-3d0d-4fac-82ee-49589d63ab89 {"id":"9d719bee-3d0d-4fac-82ee-49589d63ab89","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be86ee5f4 9d719bee-3d0d-4fac-82ee-49589d63ab89 /tidb/cdc/default/default/upstream/7365720638219039435 {"id":7365720638219039435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9d719bee-3d0d-4fac-82ee-49589d63ab89 {"id":"9d719bee-3d0d-4fac-82ee-49589d63ab89","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be86ee5f4 9d719bee-3d0d-4fac-82ee-49589d63ab89 /tidb/cdc/default/default/upstream/7365720638219039435 {"id":7365720638219039435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 11:18:14 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.59515953.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 33-th check, retry later check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ***************** properties ***************** "workload"="core" "readallfields"="true" "scanproportion"="0" "mysql.user"="root" "fieldcount"="3" "readproportion"="0" "updateproportion"="0" "operationcount"="0" "mysql.db"="consistent_replicate_nfs" "threadcount"="10" "dotransactions"="false" "insertproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="50000" "mysql.port"="4000" "requestdistribution"="uniform" ********************************************** start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 34-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:17 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 {"id":"7fb2f0dc-3e3d-4abc-beeb-9f65192822e6","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e24 7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 {"id":"7fb2f0dc-3e3d-4abc-beeb-9f65192822e6","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e24 7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 {"id":"7fb2f0dc-3e3d-4abc-beeb-9f65192822e6","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e24 7fb2f0dc-3e3d-4abc-beeb-9f65192822e6 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6004.out cli changefeed create --start-ts=449567902463164417 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365720455559486807,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:18:17.939502833+08:00","start_ts":449567902463164417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567902463164417,"checkpoint_ts":449567902463164417,"checkpoint_time":"2024-05-06 11:17:28.239"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 35-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:17.948793541+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:17.948793541+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:17.948793541+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T11:18:17.948793541+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T11:18:17.948793541+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449567902463164417 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6070.out cli changefeed remove -c changefeed-error-2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa33be80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9, pid:1187, start at 2024-05-06 11:18:19.643535158 +0800 CST m=+5.718318974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:19.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:19.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:19.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 36-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa347780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:1131, start at 2024-05-06 11:18:20.408708777 +0800 CST m=+5.283078500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa347780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:1131, start at 2024-05-06 11:18:20.408708777 +0800 CST m=+5.283078500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.415 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.382 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.382 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa347580013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:1211, start at 2024-05-06 11:18:20.407279652 +0800 CST m=+5.200058106 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa33be80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9, pid:1187, start at 2024-05-06 11:18:19.643535158 +0800 CST m=+5.718318974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:19.652 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:19.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:19.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa33bec0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-mmgh9, pid:1239, start at 2024-05-06 11:18:19.671196704 +0800 CST m=+5.666053763 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:19.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:19.643 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:19.643 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 37-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa34f100008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:1154, start at 2024-05-06 11:18:20.885418112 +0800 CST m=+6.464432603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.899 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa34f100008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:1154, start at 2024-05-06 11:18:20.885418112 +0800 CST m=+6.464432603 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.899 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa34f180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:1195, start at 2024-05-06 11:18:20.871783745 +0800 CST m=+6.372799142 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:20.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:20.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:20.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:18:23 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61086110.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa36c7c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1, pid:1205, start at 2024-05-06 11:18:22.760822082 +0800 CST m=+5.315935676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:22.768 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:22.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:22.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... [Mon May 6 11:18:24 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.24722474.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 38-th check, retry later [Mon May 6 11:18:24 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.25792581.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 11:18:24 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.25042506.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) INSERT - Takes(s): 10.0, Count: 17773, OPS: 1770.8, Avg(us): 5585, Min(us): 804, Max(us): 109472, 95th(us): 29000, 99th(us): 75000 check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa36c7c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1, pid:1205, start at 2024-05-06 11:18:22.760822082 +0800 CST m=+5.315935676 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:22.768 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:22.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:22.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa36e04000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1l5b1, pid:1293, start at 2024-05-06 11:18:22.862885391 +0800 CST m=+5.329585429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:22.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:22.849 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:22.849 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa394140002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:1188, start at 2024-05-06 11:18:25.28669031 +0800 CST m=+5.501812557 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:25.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:25.285 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:25.285 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:26 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1376a381-5e5a-45e9-8ff7-5e5025895a37 {"id":"1376a381-5e5a-45e9-8ff7-5e5025895a37","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e5d 1376a381-5e5a-45e9-8ff7-5e5025895a37 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1376a381-5e5a-45e9-8ff7-5e5025895a37 {"id":"1376a381-5e5a-45e9-8ff7-5e5025895a37","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e5d 1376a381-5e5a-45e9-8ff7-5e5025895a37 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1376a381-5e5a-45e9-8ff7-5e5025895a37 {"id":"1376a381-5e5a-45e9-8ff7-5e5025895a37","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4be7c67e5d 1376a381-5e5a-45e9-8ff7-5e5025895a37 /tidb/cdc/default/default/upstream/7365720455559486807 {"id":7365720455559486807,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6156.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 39-th check, retry later Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365720455559486807,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:18:26.841992058+08:00","start_ts":449567917785219076,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567917785219076,"checkpoint_ts":449567917785219076,"checkpoint_time":"2024-05-06 11:18:26.688"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 11:18:27 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3a6e40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:1238, start at 2024-05-06 11:18:26.51331381 +0800 CST m=+5.425722734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:26.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:26.489 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:26.489 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.25102512.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83260644-3dbb-44c6-9c5e-54e6d629c53f {"id":"83260644-3dbb-44c6-9c5e-54e6d629c53f","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a492f2 83260644-3dbb-44c6-9c5e-54e6d629c53f /tidb/cdc/default/default/upstream/7365720697278144182 {"id":7365720697278144182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.2537.out cli changefeed create '--sink-uri=mysql://normal:123456@127.0.0.1:3306?transaction-atomicity=none' < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2662.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2610.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: 03b98417-135e-4e8f-b98a-3660ff77cc3d Info: {"upstream_id":7365720697278144182,"namespace":"default","id":"03b98417-135e-4e8f-b98a-3660ff77cc3d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306?transaction-atomicity=none","create_time":"2024-05-06T11:18:27.999200655+08:00","start_ts":449567918087995398,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"transaction_atomicity":"none","encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567918087995398,"checkpoint_ts":449567918087995398,"checkpoint_time":"2024-05-06 11:18:27.843"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa394140002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:1188, start at 2024-05-06 11:18:25.28669031 +0800 CST m=+5.501812557 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:25.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:25.285 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:25.285 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa394d00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:1260, start at 2024-05-06 11:18:25.367120342 +0800 CST m=+5.516035908 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:25.375 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:25.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:25.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 54437c08-bcb7-4c27-b76d-3fa09c743d90 Info: {"upstream_id":7365720704448955381,"namespace":"default","id":"54437c08-bcb7-4c27-b76d-3fa09c743d90","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:28.122855698+08:00","start_ts":449567918137278465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567918137278465,"checkpoint_ts":449567918137278465,"checkpoint_time":"2024-05-06 11:18:28.031"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567917785219076, '"checkpoint_time":' '"2024-05-06' '11:18:26.688",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6231034d-7d37-491d-97e3-3d943af810fe {"id":"6231034d-7d37-491d-97e3-3d943af810fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a478f1 6231034d-7d37-491d-97e3-3d943af810fe /tidb/cdc/default/default/upstream/7365720691176227014 {"id":7365720691176227014,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6231034d-7d37-491d-97e3-3d943af810fe {"id":"6231034d-7d37-491d-97e3-3d943af810fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a478f1 6231034d-7d37-491d-97e3-3d943af810fe /tidb/cdc/default/default/upstream/7365720691176227014 {"id":7365720691176227014,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6231034d-7d37-491d-97e3-3d943af810fe {"id":"6231034d-7d37-491d-97e3-3d943af810fe","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a478f1 6231034d-7d37-491d-97e3-3d943af810fe /tidb/cdc/default/default/upstream/7365720691176227014 {"id":7365720691176227014,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 43c872cf-c393-4118-a469-7c09a313ffc4 Info: {"upstream_id":7365720691176227014,"namespace":"default","id":"43c872cf-c393-4118-a469-7c09a313ffc4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2024-05-06T11:18:28.291880281+08:00","start_ts":449567917321224193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567917321224193,"checkpoint_ts":449567917321224193,"checkpoint_time":"2024-05-06 11:18:24.918"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3bdc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf, pid:1305, start at 2024-05-06 11:18:27.998340992 +0800 CST m=+7.143126717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:28.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:28.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:28.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3bdc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf, pid:1305, start at 2024-05-06 11:18:27.998340992 +0800 CST m=+7.143126717 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:28.006 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:28.004 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:28.004 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3b2d80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-xw3bf, pid:1381, start at 2024-05-06 11:18:27.294676334 +0800 CST m=+6.354925244 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:27.302 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:27.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:27.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 40-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3b940000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm, pid:1247, start at 2024-05-06 11:18:27.685936899 +0800 CST m=+5.327380253 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:27.694 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:27.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:27.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3a6e40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:1238, start at 2024-05-06 11:18:26.51331381 +0800 CST m=+5.425722734 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:26.522 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:26.489 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:26.489 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3a92c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:1309, start at 2024-05-06 11:18:26.646221017 +0800 CST m=+5.482685490 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:26.654 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:26.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:26.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3cb8c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt, pid:1224, start at 2024-05-06 11:18:28.837238468 +0800 CST m=+6.287962218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:28.850 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:28.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:28.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3cb8c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt, pid:1224, start at 2024-05-06 11:18:28.837238468 +0800 CST m=+6.287962218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:28.850 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:28.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:28.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { + set +x + set +x cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "threadcount"="1" "insertproportion"="0" "updateproportion"="0" "workload"="core" "recordcount"="5000" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.port"="4000" "mysql.db"="big_txn" "dotransactions"="false" "fieldcount"="100" "scanproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "operationcount"="0" ********************************************** + set +x + tso='449567918155628545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567918155628545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x run test case test_kill_owner [Mon May 6 11:18:29 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.26672669.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later check diff failed 4-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2578.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 41-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3b940000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm, pid:1247, start at 2024-05-06 11:18:27.685936899 +0800 CST m=+5.327380253 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:27.694 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:27.664 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:27.664 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3bab40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-p68sm, pid:1327, start at 2024-05-06 11:18:27.759783765 +0800 CST m=+5.321262957 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:27.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:27.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:27.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567917785219076, '"checkpoint_time":' '"2024-05-06' '11:18:26.688",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2620.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa3ec280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-l24jt, pid:1306, start at 2024-05-06 11:18:30.927796712 +0800 CST m=+8.325564722 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:30.940 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:30.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:30.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc already exists) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table owner_resign.t1 exists + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2554.out cli tso query --pd=http://127.0.0.1:2379 TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 5-th time, retry later + set +x + tso='449567918767734785 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567918767734785 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:18:31 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26162618.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + set +x + tso='449567918864465926 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567918864465926 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:18:32 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.26602662.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2716.out cli changefeed create --start-ts=449567918155628545 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + set +x + tso='449567919109046273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567919109046273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:18:33 CST 2024] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.25882590.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2635.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: 896d0161-e516-48a6-84de-2b5d7c054965 Info: {"upstream_id":7365720710740210451,"namespace":"default","id":"896d0161-e516-48a6-84de-2b5d7c054965","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:33.35996629+08:00","start_ts":449567918155628545,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567918155628545,"checkpoint_ts":449567918155628545,"checkpoint_time":"2024-05-06 11:18:28.101"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:18:33 CST 2024] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.25652567.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 43-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc already exists) check diff successfully check diff successfully [Mon May 6 11:18:34 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28382840.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449567919443279874 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567919443279874 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:18:34 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.26722674.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567917785219076, '"checkpoint_time":' '"2024-05-06' '11:18:26.688",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe3880a-b91a-4227-937d-3854ab1e8bc8 {"id":"efe3880a-b91a-4227-937d-3854ab1e8bc8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bd8bf6 efe3880a-b91a-4227-937d-3854ab1e8bc8 /tidb/cdc/default/default/upstream/7365720720694283637 {"id":7365720720694283637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe3880a-b91a-4227-937d-3854ab1e8bc8 {"id":"efe3880a-b91a-4227-937d-3854ab1e8bc8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bd8bf6 efe3880a-b91a-4227-937d-3854ab1e8bc8 /tidb/cdc/default/default/upstream/7365720720694283637 {"id":7365720720694283637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/efe3880a-b91a-4227-937d-3854ab1e8bc8 {"id":"efe3880a-b91a-4227-937d-3854ab1e8bc8","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bd8bf6 efe3880a-b91a-4227-937d-3854ab1e8bc8 /tidb/cdc/default/default/upstream/7365720720694283637 {"id":7365720720694283637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2683.out cli changefeed create --start-ts=449567918767734785 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0034f536-6bc4-4d80-bae3-a001b03b678b Info: {"upstream_id":7365720720694283637,"namespace":"default","id":"0034f536-6bc4-4d80-bae3-a001b03b678b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:35.421008393+08:00","start_ts":449567918767734785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567918767734785,"checkpoint_ts":449567918767734785,"checkpoint_time":"2024-05-06 11:18:30.436"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... INSERT - Takes(s): 20.0, Count: 27874, OPS: 1394.2, Avg(us): 7140, Min(us): 804, Max(us): 328397, 95th(us): 66000, 99th(us): 82000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bdbc09-6124-465e-9850-66bac6dded05 {"id":"36bdbc09-6124-465e-9850-66bac6dded05","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bdcff4 36bdbc09-6124-465e-9850-66bac6dded05 /tidb/cdc/default/default/upstream/7365720726453112572 {"id":7365720726453112572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bdbc09-6124-465e-9850-66bac6dded05 {"id":"36bdbc09-6124-465e-9850-66bac6dded05","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bdcff4 36bdbc09-6124-465e-9850-66bac6dded05 /tidb/cdc/default/default/upstream/7365720726453112572 {"id":7365720726453112572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bdbc09-6124-465e-9850-66bac6dded05 {"id":"36bdbc09-6124-465e-9850-66bac6dded05","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8bdcff4 36bdbc09-6124-465e-9850-66bac6dded05 /tidb/cdc/default/default/upstream/7365720726453112572 {"id":7365720726453112572,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2706.out cli changefeed create --start-ts=449567918864465926 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ [Pipeline] } Create changefeed successfully! ID: 25a300df-4f25-4e37-8de4-1c3d2125fbb4 Info: {"upstream_id":7365720726453112572,"namespace":"default","id":"25a300df-4f25-4e37-8de4-1c3d2125fbb4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:35.820811839+08:00","start_ts":449567918864465926,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567918864465926,"checkpoint_ts":449567918864465926,"checkpoint_time":"2024-05-06 11:18:30.805"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:36 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17ecb263-621f-4a76-bdce-7d12910f7720 {"id":"17ecb263-621f-4a76-bdce-7d12910f7720","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8be85fc 17ecb263-621f-4a76-bdce-7d12910f7720 /tidb/cdc/default/default/upstream/7365720727244065389 {"id":7365720727244065389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17ecb263-621f-4a76-bdce-7d12910f7720 {"id":"17ecb263-621f-4a76-bdce-7d12910f7720","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8be85fc 17ecb263-621f-4a76-bdce-7d12910f7720 /tidb/cdc/default/default/upstream/7365720727244065389 {"id":7365720727244065389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17ecb263-621f-4a76-bdce-7d12910f7720 {"id":"17ecb263-621f-4a76-bdce-7d12910f7720","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8be85fc 17ecb263-621f-4a76-bdce-7d12910f7720 /tidb/cdc/default/default/upstream/7365720727244065389 {"id":7365720727244065389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2641.out cli changefeed create --start-ts=449567919109046273 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later "is-owner": true, run task successfully Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7365720727244065389,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T11:18:36.774413223+08:00","start_ts":449567919109046273,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567919109046273,"checkpoint_ts":449567919109046273,"checkpoint_time":"2024-05-06 11:18:31.738"} PASS + set +x coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 44-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:36 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75e8c5fd-17ed-4cff-82c2-5cf0403a586b {"id":"75e8c5fd-17ed-4cff-82c2-5cf0403a586b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b518fa 75e8c5fd-17ed-4cff-82c2-5cf0403a586b /tidb/cdc/default/default/upstream/7365720717550919806 {"id":7365720717550919806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75e8c5fd-17ed-4cff-82c2-5cf0403a586b {"id":"75e8c5fd-17ed-4cff-82c2-5cf0403a586b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b518fa 75e8c5fd-17ed-4cff-82c2-5cf0403a586b /tidb/cdc/default/default/upstream/7365720717550919806 {"id":7365720717550919806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75e8c5fd-17ed-4cff-82c2-5cf0403a586b {"id":"75e8c5fd-17ed-4cff-82c2-5cf0403a586b","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b518fa 75e8c5fd-17ed-4cff-82c2-5cf0403a586b /tidb/cdc/default/default/upstream/7365720717550919806 {"id":7365720717550919806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 49e5f35e-8164-4d39-ba80-4f31290851d5 Info: {"upstream_id":7365720717550919806,"namespace":"default","id":"49e5f35e-8164-4d39-ba80-4f31290851d5","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:36.915821615+08:00","start_ts":449567919553904644,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567919553904644,"checkpoint_ts":449567919553904644,"checkpoint_time":"2024-05-06 11:18:33.435"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/54437c08-bcb7-4c27-b76d-3fa09c743d90 {UpstreamID:7365720704448955381 Namespace:default ID:54437c08-bcb7-4c27-b76d-3fa09c743d90 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:28.122855698 +0800 CST StartTs:449567918137278465 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cfcd80 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567918150385670} {CheckpointTs:449567920273752077 MinTableBarrierTs:449567920273752077 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5dc900-9e0a-4484-95be-f3fb826a6cc9 {"id":"0d5dc900-9e0a-4484-95be-f3fb826a6cc9","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a58263 0d5dc900-9e0a-4484-95be-f3fb826a6cc9 /tidb/cdc/default/default/changefeed/info/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"upstream-id":7365720704448955381,"namespace":"default","changefeed-id":"54437c08-bcb7-4c27-b76d-3fa09c743d90","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:28.122855698+08:00","start-ts":449567918137278465,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567918150385670} /tidb/cdc/default/default/changefeed/status/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":449567920273752077,"min-table-barrier-ts":449567920273752077,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d5dc900-9e0a-4484-95be-f3fb826a6cc9/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/29722779-692f-42b6-9e4f-57c0bb1b3708/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/54437c08-bcb7-4c27-b76d-3fa09c743d90 {UpstreamID:7365720704448955381 Namespace:default ID:54437c08-bcb7-4c27-b76d-3fa09c743d90 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:28.122855698 +0800 CST StartTs:449567918137278465 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cfcd80 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567918150385670} {CheckpointTs:449567920273752077 MinTableBarrierTs:449567920273752077 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5dc900-9e0a-4484-95be-f3fb826a6cc9 {"id":"0d5dc900-9e0a-4484-95be-f3fb826a6cc9","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a58263 0d5dc900-9e0a-4484-95be-f3fb826a6cc9 /tidb/cdc/default/default/changefeed/info/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"upstream-id":7365720704448955381,"namespace":"default","changefeed-id":"54437c08-bcb7-4c27-b76d-3fa09c743d90","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:28.122855698+08:00","start-ts":449567918137278465,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567918150385670} /tidb/cdc/default/default/changefeed/status/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":449567920273752077,"min-table-barrier-ts":449567920273752077,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d5dc900-9e0a-4484-95be-f3fb826a6cc9/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/29722779-692f-42b6-9e4f-57c0bb1b3708/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/54437c08-bcb7-4c27-b76d-3fa09c743d90 {UpstreamID:7365720704448955381 Namespace:default ID:54437c08-bcb7-4c27-b76d-3fa09c743d90 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:28.122855698 +0800 CST StartTs:449567918137278465 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cfcd80 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567918150385670} {CheckpointTs:449567920273752077 MinTableBarrierTs:449567920273752077 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d5dc900-9e0a-4484-95be-f3fb826a6cc9 {"id":"0d5dc900-9e0a-4484-95be-f3fb826a6cc9","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/29722779-692f-42b6-9e4f-57c0bb1b3708 {"id":"29722779-692f-42b6-9e4f-57c0bb1b3708","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a581f0 29722779-692f-42b6-9e4f-57c0bb1b3708 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8a58263 0d5dc900-9e0a-4484-95be-f3fb826a6cc9 /tidb/cdc/default/default/changefeed/info/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"upstream-id":7365720704448955381,"namespace":"default","changefeed-id":"54437c08-bcb7-4c27-b76d-3fa09c743d90","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:28.122855698+08:00","start-ts":449567918137278465,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567918150385670} /tidb/cdc/default/default/changefeed/status/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":449567920273752077,"min-table-barrier-ts":449567920273752077,"admin-job-type":0} /tidb/cdc/default/default/task/position/0d5dc900-9e0a-4484-95be-f3fb826a6cc9/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/29722779-692f-42b6-9e4f-57c0bb1b3708/54437c08-bcb7-4c27-b76d-3fa09c743d90 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720704448955381 {"id":7365720704448955381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 owner pid: 2672 owner id cbb87fb6-6507-40d4-9ace-39613f625455 [Mon May 6 11:18:37 CST 2024] <<<<<< START cdc server in availability case >>>>>> go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.28162818.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:37 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13ac8878-e737-41ab-9d59-ab93b9cbe7e4 {"id":"13ac8878-e737-41ab-9d59-ab93b9cbe7e4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8c79bf2 13ac8878-e737-41ab-9d59-ab93b9cbe7e4 /tidb/cdc/default/default/upstream/7365720735893246412 {"id":7365720735893246412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13ac8878-e737-41ab-9d59-ab93b9cbe7e4 {"id":"13ac8878-e737-41ab-9d59-ab93b9cbe7e4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8c79bf2 13ac8878-e737-41ab-9d59-ab93b9cbe7e4 /tidb/cdc/default/default/upstream/7365720735893246412 {"id":7365720735893246412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13ac8878-e737-41ab-9d59-ab93b9cbe7e4 {"id":"13ac8878-e737-41ab-9d59-ab93b9cbe7e4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8c79bf2 13ac8878-e737-41ab-9d59-ab93b9cbe7e4 /tidb/cdc/default/default/upstream/7365720735893246412 {"id":7365720735893246412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2723.out cli changefeed create --start-ts=449567919443279874 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 88bcb818-6c98-4fd7-a095-ff46b1d3b536 Info: {"upstream_id":7365720735893246412,"namespace":"default","id":"88bcb818-6c98-4fd7-a095-ff46b1d3b536","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:18:37.941872367+08:00","start_ts":449567919443279874,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567919443279874,"checkpoint_ts":449567919443279874,"checkpoint_time":"2024-05-06 11:18:33.013"} PASS table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa439e00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7, pid:1286, start at 2024-05-06 11:18:35.922555792 +0800 CST m=+6.397876521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:35.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:35.896 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:35.896 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa439e00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7, pid:1286, start at 2024-05-06 11:18:35.922555792 +0800 CST m=+6.397876521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:35.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:35.896 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:35.896 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa439a00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-ch6s7, pid:1364, start at 2024-05-06 11:18:35.923792391 +0800 CST m=+6.303433450 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:20:35.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:18:35.930 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:08:35.930 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7365720727244065389, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:36.78386657+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7365720727244065389, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:36.78386657+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7365720727244065389, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-05-06T11:18:36.78386657+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7365720727244065389, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T11:18:36.78386657+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365720727244065389, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-05-06T11:18:36.78386657+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully go: downloading golang.org/x/text v0.14.0 table ddl_manager.finish_mark not exists for 45-th check, retry later check diff failed 2-th time, retry later + set +x go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... [Pipeline] // cache [Pipeline] } go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/coreos/go-semver v0.3.1 table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) go: downloading golang.org/x/text v0.14.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:18:40 CST 2024] <<<<<< run test case changefeed_fast_fail success! >>>>>> go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.6 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2622.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later [Pipeline] // cache [Pipeline] } table ddl_manager.finish_mark not exists for 46-th check, retry later go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033a6630 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567918601273354 MinTableBarrierTs:449567921419321349 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ab23c2c-7618-4134-a064-3aae58929472 {"id":"8ab23c2c-7618-4134-a064-3aae58929472","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0855f 8ab23c2c-7618-4134-a064-3aae58929472 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567918601273354,"min-table-barrier-ts":449567921419321349,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ab23c2c-7618-4134-a064-3aae58929472/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbb87fb6-6507-40d4-9ace-39613f625455/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033a6630 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567918601273354 MinTableBarrierTs:449567921419321349 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ab23c2c-7618-4134-a064-3aae58929472 {"id":"8ab23c2c-7618-4134-a064-3aae58929472","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0855f 8ab23c2c-7618-4134-a064-3aae58929472 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567918601273354,"min-table-barrier-ts":449567921419321349,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ab23c2c-7618-4134-a064-3aae58929472/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbb87fb6-6507-40d4-9ace-39613f625455/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0033a6630 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567918601273354 MinTableBarrierTs:449567921419321349 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ab23c2c-7618-4134-a064-3aae58929472 {"id":"8ab23c2c-7618-4134-a064-3aae58929472","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/cbb87fb6-6507-40d4-9ace-39613f625455 {"id":"cbb87fb6-6507-40d4-9ace-39613f625455","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08515 cbb87fb6-6507-40d4-9ace-39613f625455 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0855f 8ab23c2c-7618-4134-a064-3aae58929472 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567918601273354,"min-table-barrier-ts":449567921419321349,"admin-job-type":0} /tidb/cdc/default/default/task/position/8ab23c2c-7618-4134-a064-3aae58929472/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cbb87fb6-6507-40d4-9ace-39613f625455/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "cbb87fb6-6507-40d4-9ace-39613f625455" | grep -v cluster_id | grep id go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da [Pipeline] // withCredentials [Pipeline] } go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567917785219076, '"checkpoint_time":' '"2024-05-06' '11:18:26.688",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 4-th time, retry later go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 + set +x + tso='449567921365581825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567921365581825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:18:41 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.26602662.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "cluster-id": "default" "id": "8ab23c2c-7618-4134-a064-3aae58929472", "cluster-id": "default" run task successfully INSERT - Takes(s): 10.0, Count: 2014, OPS: 201.1, Avg(us): 4630, Min(us): 1620, Max(us): 88321, 95th(us): 9000, 99th(us): 66000 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 not exists for 1-th check, retry later check diff failed 2-th time, retry later go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 table ddl_manager.finish_mark not exists for 47-th check, retry later go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kylelemons/godebug v1.1.0 [Pipeline] // dir [Pipeline] } check diff failed 4-th time, retry later go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 check diff successfully check diff failed 2-th time, retry later go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da [Pipeline] // dir [Pipeline] } go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 wait process cdc.test exit for 1-th time... [Pipeline] // timeout [Pipeline] } go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 wait process cdc.test exit for 2-th time... capture_id: 8ab23c2c-7618-4134-a064-3aae58929472 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN wait process 2672 exit for 1-th time... table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 3-th time, retry later [Pipeline] // withCredentials cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:18:44 CST 2024] <<<<<< run test case tiflash success! >>>>>> [Pipeline] } table ddl_manager.finish_mark not exists for 48-th check, retry later check diff failed 5-th time, retry later wait process 2672 exit for 2-th time... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // stage /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2672) - No such process wait process 2672 exit for 3-th time... process 2672 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 8ab23c2c-7618-4134-a064-3aae58929472 -A1 | grep '"is-owner": true' [Pipeline] } check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 [Pipeline] { [Pipeline] { + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:44 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34587202-2e48-4454-aff2-10030d5e153d {"id":"34587202-2e48-4454-aff2-10030d5e153d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8e279f5 34587202-2e48-4454-aff2-10030d5e153d /tidb/cdc/default/default/upstream/7365720764667538355 {"id":7365720764667538355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34587202-2e48-4454-aff2-10030d5e153d {"id":"34587202-2e48-4454-aff2-10030d5e153d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8e279f5 34587202-2e48-4454-aff2-10030d5e153d /tidb/cdc/default/default/upstream/7365720764667538355 {"id":7365720764667538355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/34587202-2e48-4454-aff2-10030d5e153d {"id":"34587202-2e48-4454-aff2-10030d5e153d","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8e279f5 34587202-2e48-4454-aff2-10030d5e153d /tidb/cdc/default/default/upstream/7365720764667538355 {"id":7365720764667538355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2721.out cli changefeed create --start-ts=449567921365581825 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic go: downloading github.com/jmespath/go-jmespath v0.4.0 Create changefeed successfully! ID: 547757a5-d90c-4edc-8c4b-d8e4b38b8eb3 Info: {"upstream_id":7365720764667538355,"namespace":"default","id":"547757a5-d90c-4edc-8c4b-d8e4b38b8eb3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2024-05-06T11:18:45.364660835+08:00","start_ts":449567921365581825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567921365581825,"checkpoint_ts":449567921365581825,"checkpoint_time":"2024-05-06 11:18:40.346"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // timeout INSERT - Takes(s): 30.0, Count: 39987, OPS: 1333.1, Avg(us): 7468, Min(us): 804, Max(us): 328397, 95th(us): 69000, 99th(us): 83000 [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 4-th time, retry later check diff failed 2-th time, retry later [Pipeline] // container go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table ddl_manager.finish_mark not exists for 49-th check, retry later check diff failed 6-th time, retry later + set +x go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 [Pipeline] } go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pkg/errors v0.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sys v0.19.0 [Pipeline] // withEnv [Pipeline] } go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow [Pipeline] { "is-owner": true, run task successfully test_kill_owner: pass go: downloading golang.org/x/text v0.14.0 [Pipeline] // container [Pipeline] } [Pipeline] cache go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later check diff failed 5-th time, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 50-th check, retry later check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_owner [Mon May 6 11:18:49 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.29922994.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 check diff failed 4-th time, retry later go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/time v0.5.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/dolthub/maphash v0.1.0 check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449567917785219076, "checkpoint_time": "2024-05-06 11:18:26.688", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449567917785219076, '"checkpoint_time":' '"2024-05-06' '11:18:26.688",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 5-th time, retry later INSERT - Takes(s): 20.0, Count: 3739, OPS: 187.0, Avg(us): 4994, Min(us): 1620, Max(us): 90255, 95th(us): 11000, 99th(us): 67000 wait process cdc.test exit for 3-th time... go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 table ddl_manager.finish_mark not exists for 51-th check, retry later go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 check diff successfully check diff failed 1-th time, retry later go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 11:18:51 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd table ddl_manager.finish_mark not exists for 52-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:18:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002f1ebd0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567921943609346 MinTableBarrierTs:449567924302905349 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567924565049347, checkpointTs: 449567921943609346, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567921943609346,"min-table-barrier-ts":449567924302905349,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002f1ebd0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567921943609346 MinTableBarrierTs:449567924302905349 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567924565049347, checkpointTs: 449567921943609346, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567921943609346,"min-table-barrier-ts":449567924302905349,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"+ grep -q 'failed to get info:' pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002f1ebd0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567921943609346 MinTableBarrierTs:449567924302905349 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567924565049349, checkpointTs: 449567921943609346, state: Prepared span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567924565049347, checkpointTs: 449567921943609346, state: Prepared *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567921943609346,"min-table-barrier-ts":449567924302905349,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' check diff successfully check diff failed 5-th time, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... "is-owner": true, run task successfully Run finished, takes 38.159718416s INSERT - Takes(s): 38.2, Count: 50000, OPS: 1310.5, Avg(us): 7572, Min(us): 804, Max(us): 328397, 95th(us): 71000, 99th(us): 84000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 53-th check, retry later check diff failed 6-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 11:18:54 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:18:56 CST 2024] <<<<<< run test case owner_resign success! >>>>>> check diff failed 7-th time, retry later Run finished, takes 27.069041425s INSERT - Takes(s): 27.1, Count: 5000, OPS: 184.8, Avg(us): 5053, Min(us): 1620, Max(us): 90255, 95th(us): 13000, 99th(us): 68000 table big_txn.usertable exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 54-th check, retry later [2024/05/06 11:18:57.554 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001c16380/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/05/06 11:18:58.633 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0025701c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"] Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later owner pid: 2997 owner id 219e8958-5da5-445a-96bb-ec17c1eac942 [Mon May 6 11:18:59 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.31093111.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 8-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926082076680, "checkpoint_time": "2024-05-06 11:18:58.338", "error": null }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926082076680, "checkpoint_time": "2024-05-06 11:18:58.338", "error": null }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926082076680, "checkpoint_time": "2024-05-06 11:18:58.338", "error": null } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449567926082076680, '"checkpoint_time":' '"2024-05-06' '11:18:58.338",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449567926082076680, '"checkpoint_time":' '"2024-05-06' '11:18:58.338",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6414.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 8.47 secs (397788411 bytes/sec) [Pipeline] { [Pipeline] cache + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": { "time": "2024-05-06T11:18:27.2190929+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449567926344220677, '"checkpoint_time":' '"2024-05-06' '11:18:59.338",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449567926344220677, '"checkpoint_time":' '"2024-05-06' '11:18:59.338",' '"error":' '{' '"time":' '"2024-05-06T11:18:27.2190929+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6473.out cli changefeed resume -c changefeed-initialize-error PASS ____________________________________ *************************** 1. row *************************** primary_ts: 449567863260577793 secondary_ts: 449567865893289998 *************************** 2. row *************************** primary_ts: 449567871124897792 secondary_ts: 449567871555600385 *************************** 3. row *************************** primary_ts: 449567878989217792 secondary_ts: 449567879158038529 *************************** 4. row *************************** primary_ts: 449567886853537792 secondary_ts: 449567887048310785 *************************** 5. row *************************** primary_ts: 449567894717857792 secondary_ts: 449567894912630785 *************************** 6. row *************************** primary_ts: 449567902582177792 secondary_ts: 449567902855593985 *************************** 7. row *************************** primary_ts: 449567910446497792 secondary_ts: 449567910719913985 *************************** 8. row *************************** primary_ts: 449567918310817792 secondary_ts: 449567918584496129 skip invalid syncpoint primary_ts: 449567863260577793, first_ddl_ts: 449567866733985809 check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:19:01 CST 2024] <<<<<< run test case syncpoint success! >>>>>> coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567926924345351 MinTableBarrierTs:449567926924345354 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/534a9793-6dbf-4c0b-bcda-79157b1357fe {"id":"534a9793-6dbf-4c0b-bcda-79157b1357fe","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0860f 534a9793-6dbf-4c0b-bcda-79157b1357fe /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567926924345351,"min-table-barrier-ts":449567926924345354,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/534a9793-6dbf-4c0b-bcda-79157b1357fe/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567926924345351 MinTableBarrierTs:449567926924345354 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/534a9793-6dbf-4c0b-bcda-79157b1357fe {"id":"534a9793-6dbf-4c0b-bcda-79157b1357fe","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0860f 534a9793-6dbf-4c0b-bcda-79157b1357fe /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567926924345351,"min-table-barrier-ts":449567926924345354,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/534a9793-6dbf-4c0b-bcda-79157b1357fe/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00309eea0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567926924345351 MinTableBarrierTs:449567926924345354 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219e8958-5da5-445a-96bb-ec17c1eac942 {"id":"219e8958-5da5-445a-96bb-ec17c1eac942","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/534a9793-6dbf-4c0b-bcda-79157b1357fe {"id":"534a9793-6dbf-4c0b-bcda-79157b1357fe","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b085a8 219e8958-5da5-445a-96bb-ec17c1eac942 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0860f 534a9793-6dbf-4c0b-bcda-79157b1357fe /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567926924345351,"min-table-barrier-ts":449567926924345354,"admin-job-type":0} /tidb/cdc/default/default/task/position/219e8958-5da5-445a-96bb-ec17c1eac942/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/534a9793-6dbf-4c0b-bcda-79157b1357fe/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "219e8958-5da5-445a-96bb-ec17c1eac942" | grep id check diff failed 9-th time, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s "cluster-id": "default" "id": "534a9793-6dbf-4c0b-bcda-79157b1357fe", "cluster-id": "default" + info='{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": null }' + echo '{ "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": null }' { "upstream_id": 7365720455559486807, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449567926344220677, "checkpoint_time": "2024-05-06 11:18:59.338", "error": null } ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449567926344220677, '"checkpoint_time":' '"2024-05-06' '11:18:59.338",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720455559486807, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449567926344220677, '"checkpoint_time":' '"2024-05-06' '11:18:59.338",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6535.out cli changefeed remove -c changefeed-initialize-error run task successfully check diff failed 10-th time, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449567926344220677 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa5ec840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:3709, start at 2024-05-06 11:19:03.742895838 +0800 CST m=+5.240143014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:03.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:03.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:03.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 11-th time, retry later + set +x table ddl_manager.finish_mark not exists for 59-th check, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa5ec840012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:3709, start at 2024-05-06 11:19:03.742895838 +0800 CST m=+5.240143014 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:03.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:03.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:03.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa5eec40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-s91hs, pid:3788, start at 2024-05-06 11:19:03.904290093 +0800 CST m=+5.348383390 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:03.913 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:03.908 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:03.908 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore wait process cdc.test exit for 2-th time... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 534a9793-6dbf-4c0b-bcda-79157b1357fe ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '534a9793-6dbf-4c0b-bcda-79157b1357fe' 534a9793-6dbf-4c0b-bcda-79157b1357fe run task successfully test_hang_up_owner: pass start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:19:07 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> wait process cdc.test exit for 1-th time... # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "449567924384432151" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2024/05/06 11:19:05.871 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2024/05/06 11:19:05.871 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2024/05/06 11:19:05.872 +08:00] [INFO] [redo.go:163] ["apply redo log starts"] [checkpointTs=449567923986497556] [resolvedTs=449567924384432151] [2024/05/06 11:19:05.872 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=263.67ยตs] [2024/05/06 11:19:05.873 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:19:05.873 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 11:19:05.878 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:19:05.881 +08:00] [INFO] [mysql.go:198] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2024/05/06 11:19:05.881 +08:00] [INFO] [worker.go:105] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2024/05/06 11:19:05.882 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2024/05/06 11:19:05.882 +08:00] [WARN] [config.go:380] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2024/05/06 11:19:05.888 +08:00] [INFO] [db_helper.go:175] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2024/05/06 11:19:05.890 +08:00] [INFO] [mysql_ddl_sink.go:99] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2024/05/06 11:19:05.920 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=48.132013ms] [2024/05/06 11:19:06.275 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span={table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}] [checkpointTs=449567924384432151] [2024/05/06 11:19:06.275 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span={table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}] [checkpointTs=449567924384432151] [2024/05/06 11:19:06.375 +08:00] [INFO] [redo.go:219] ["apply redo log finishes"] [appliedLogCount=2608] [appliedDDLCount=0] [currentCheckpoint=449567924384432151] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2024/05/06 11:19:06.375 +08:00] [INFO] [worker.go:120] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] Apply redo log successfully check diff successfully [Mon May 6 11:19:06 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 48341 wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) process cdc.test already exit run test case test_expire_owner [Mon May 6 11:19:08 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.32433245.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 60-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:19:10 CST 2024] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.50635065.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 12-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035f7440 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567929283379208 MinTableBarrierTs:449567929545785352 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567929545785349, checkpointTs: 449567929545785349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc {"id":"7f4c17fe-d4f1-4c8d-9896-d8097c492dcc","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08655 7f4c17fe-d4f1-4c8d-9896-d8097c492dcc /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567929283379208,"min-table-barrier-ts":449567929545785352,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035f7440 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567929283379208 MinTableBarrierTs:449567929545785352 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567929545785349, checkpointTs: 449567929545785349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc {"id":"7f4c17fe-d4f1-4c8d-9896-d8097c492dcc","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08655 7f4c17fe-d4f1-4c8d-9896-d8097c492dcc /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567929283379208,"min-table-barrier-ts":449567929545785352,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0035f7440 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567929283379208 MinTableBarrierTs:449567929545785352 AdminJobType:noop} span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567929545785352, checkpointTs: 449567929545785352, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567929545785349, checkpointTs: 449567929545785349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc {"id":"7f4c17fe-d4f1-4c8d-9896-d8097c492dcc","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08655 7f4c17fe-d4f1-4c8d-9896-d8097c492dcc /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567929283379208,"min-table-barrier-ts":449567929545785352,"admin-job-type":0} /tidb/cdc/default/default/task/position/7f4c17fe-d4f1-4c8d-9896-d8097c492dcc/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: mysql... <<================= [Mon May 6 11:19:11 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark exists check diff successfully check diff failed 13-th time, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:14 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d984f1a-14a4-4a94-85ae-596cdb4fc965 {"id":"0d984f1a-14a4-4a94-85ae-596cdb4fc965","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be952dff4 0d984f1a-14a4-4a94-85ae-596cdb4fc965 /tidb/cdc/default/default/upstream/7365720886656298615 {"id":7365720886656298615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d984f1a-14a4-4a94-85ae-596cdb4fc965 {"id":"0d984f1a-14a4-4a94-85ae-596cdb4fc965","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be952dff4 0d984f1a-14a4-4a94-85ae-596cdb4fc965 /tidb/cdc/default/default/upstream/7365720886656298615 {"id":7365720886656298615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0d984f1a-14a4-4a94-85ae-596cdb4fc965 {"id":"0d984f1a-14a4-4a94-85ae-596cdb4fc965","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be952dff4 0d984f1a-14a4-4a94-85ae-596cdb4fc965 /tidb/cdc/default/default/upstream/7365720886656298615 {"id":7365720886656298615,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5118.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Verifying downstream PD is started... Create changefeed successfully! ID: test Info: {"upstream_id":7365720886656298615,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T11:19:14.444998603+08:00","start_ts":449567930270089221,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567930270089221,"checkpoint_ts":449567930270089221,"checkpoint_time":"2024-05-06 11:19:14.314"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... owner pid: 3248 owner id 7f4c17fe-d4f1-4c8d-9896-d8097c492dcc process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '7f4c17fe-d4f1-4c8d-9896-d8097c492dcc' /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08655 run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa69180000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:9506, start at 2024-05-06 11:19:14.286546812 +0800 CST m=+5.831845692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:14.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:14.272 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:14.272 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 4-th time... check diff failed 14-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6742c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:3773, start at 2024-05-06 11:19:12.42492179 +0800 CST m=+5.464635256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:12.432 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:12.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:12.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6742c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:3773, start at 2024-05-06 11:19:12.42492179 +0800 CST m=+5.464635256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:12.432 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:12.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:12.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa675880013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-cksv8, pid:3853, start at 2024-05-06 11:19:12.502689992 +0800 CST m=+5.461449838 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:12.509 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:12.482 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:12.482 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 5-th time... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x wait process cdc.test exit for 6-th time... "is-owner": true, run task successfully test_expire_owner pass wait process cdc.test exit for 7-th time... wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa69180000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:9506, start at 2024-05-06 11:19:14.286546812 +0800 CST m=+5.831845692 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:14.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:14.272 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:14.272 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6907c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-tvx5l, pid:9556, start at 2024-05-06 11:19:14.255098459 +0800 CST m=+5.740975961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:14.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:14.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:14.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Mon May 6 11:19:16 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> wait process cdc.test exit for 2-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5158.out cli changefeed pause -c test PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Mon May 6 11:19:17 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.34103412.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Mon May 6 11:19:17 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.51695171.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 15-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.10796.out cli tso query --pd=http://127.0.0.1:2379 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5188.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} Update changefeed config successfully! ID: test Info: {"upstream_id":7365720886656298615,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2024-05-06T11:19:14.444998603+08:00","start_ts":449567930270089221,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567930702626856,"checkpoint_ts":449567930702626856,"checkpoint_time":"2024-05-06 11:19:15.964"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6dbe00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:3687, start at 2024-05-06 11:19:19.069550211 +0800 CST m=+5.341553829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:19.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:19.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:19.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff failed 16-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:20 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19aa7a69-707a-4e95-9aa5-c37bf5c2a833 {"id":"19aa7a69-707a-4e95-9aa5-c37bf5c2a833","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be977a843 19aa7a69-707a-4e95-9aa5-c37bf5c2a833 /tidb/cdc/default/default/upstream/7365720921494746827 {"id":7365720921494746827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19aa7a69-707a-4e95-9aa5-c37bf5c2a833 {"id":"19aa7a69-707a-4e95-9aa5-c37bf5c2a833","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be977a843 19aa7a69-707a-4e95-9aa5-c37bf5c2a833 /tidb/cdc/default/default/upstream/7365720921494746827 {"id":7365720921494746827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/19aa7a69-707a-4e95-9aa5-c37bf5c2a833 {"id":"19aa7a69-707a-4e95-9aa5-c37bf5c2a833","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be977a843 19aa7a69-707a-4e95-9aa5-c37bf5c2a833 /tidb/cdc/default/default/upstream/7365720921494746827 {"id":7365720921494746827,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.5214.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ + set +x + tso='449567931517632513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567931517632513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:19:20 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1084310845.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c086c0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567930594361349 MinTableBarrierTs:449567931918188546 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567931905081349, checkpointTs: 449567930594361349, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567930594361349,"min-table-barrier-ts":449567931918188546,"admin-job-type":0} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c086c0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567930594361349 MinTableBarrierTs:449567931918188546 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567931905081349, checkpointTs: 449567930594361349, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567930594361349,"min-table-barrier-ts":449567931918188546,"admin-job-type":0} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c086c0 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567930594361349 MinTableBarrierTs:449567931918188546 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449567931905081349, checkpointTs: 449567930594361349, state: Replicating span: {table_id:102,start_key:7480000000000000ff665f720000000000fa,end_key:7480000000000000ff665f730000000000fa}, resolvedTs: 449567931918188546, checkpointTs: 449567930594361349, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567930594361349,"min-table-barrier-ts":449567931918188546,"admin-job-type":0} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":736572071074+ grep -q 'etcd info' 0210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5220.out cli changefeed resume -c test Create changefeed successfully! ID: 9b71f47f-731d-47a6-b76b-f60994d4ed71 Info: {"upstream_id":7365720921494746827,"namespace":"default","id":"9b71f47f-731d-47a6-b76b-f60994d4ed71","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:19:20.981427528+08:00","start_ts":449567931982151685,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567931982151685,"checkpoint_ts":449567931982151685,"checkpoint_time":"2024-05-06 11:19:20.845"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6dbe00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:3687, start at 2024-05-06 11:19:19.069550211 +0800 CST m=+5.341553829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:19.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:19.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:19.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa6ddc8000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-jlfnx, pid:3736, start at 2024-05-06 11:19:19.168770623 +0800 CST m=+5.381765634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:19.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:19.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:19.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } "is-owner": true, run task successfully check diff failed 17-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... owner pid: 3415 owner id f974e8e4-15a5-4c5d-9c4a-3e9056e8818c [Mon May 6 11:19:23 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35203522.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:23 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2912e052-62d7-4baa-a607-fefc5f5a2e73 {"id":"2912e052-62d7-4baa-a607-fefc5f5a2e73","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be97d3a50 2912e052-62d7-4baa-a607-fefc5f5a2e73 /tidb/cdc/default/default/upstream/7365720931959301550 {"id":7365720931959301550,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2912e052-62d7-4baa-a607-fefc5f5a2e73 {"id":"2912e052-62d7-4baa-a607-fefc5f5a2e73","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be97d3a50 2912e052-62d7-4baa-a607-fefc5f5a2e73 /tidb/cdc/default/default/upstream/7365720931959301550 {"id":7365720931959301550,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2912e052-62d7-4baa-a607-fefc5f5a2e73 {"id":"2912e052-62d7-4baa-a607-fefc5f5a2e73","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be97d3a50 2912e052-62d7-4baa-a607-fefc5f5a2e73 /tidb/cdc/default/default/upstream/7365720931959301550 {"id":7365720931959301550,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.10895.out cli changefeed create --start-ts=449567931517632513 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365720931959301550,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:19:24.276942391+08:00","start_ts":449567931517632513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567931517632513,"checkpoint_ts":449567931517632513,"checkpoint_time":"2024-05-06 11:19:19.073"} PASS check diff failed 18-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa722140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:4007, start at 2024-05-06 11:19:23.542652697 +0800 CST m=+5.289260255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:23.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:23.525 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:23.525 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + set +x [Mon May 6 11:19:25 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.50935095.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table test.simple not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5254.out cli changefeed pause -c test split_and_random_merge scale: 20 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359/tiflow-cdc) 3368429056 bytes in 20.35 secs (165522553 bytes/sec) [Pipeline] { check diff failed 19-th time, retry later [Pipeline] sh + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:26 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030fd680 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567933215801355 MinTableBarrierTs:449567933228908547 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567933215801355,"min-table-barrier-ts":449567933490790403,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030fd680 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567933215801355 MinTableBarrierTs:449567933228908547 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567933215801355,"min-table-barrier-ts":449567933490790403,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030fd680 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567933215801355 MinTableBarrierTs:449567933228908547 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567933215801355,"min-table-barrier-ts":449567933490790403,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "f974e8e4-15a5-4c5d-9c4a-3e9056e8818c" | grep id =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa722140010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:4007, start at 2024-05-06 11:19:23.542652697 +0800 CST m=+5.289260255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:23.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:23.525 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:23.525 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa722040013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-j25jc, pid:4088, start at 2024-05-06 11:19:23.542910936 +0800 CST m=+5.220731143 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:23.549 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:23.521 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:23.521 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... [Pipeline] sh Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=eee40e20-ad93-43a5-9f61-f983bac5992e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-n9hfw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-n9hfw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // container [Pipeline] } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5280.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=2e28660c-a971-4d93-b4a6-5e6256b7b7d3 JENKINS_NODE_COOKIE=fb075230-fa82-46f2-8240-a35ba01aceb8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/ GOLANG_VERSION=1.21.6 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_mysql_test-359 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test","buildid":"1787315699467685890","prowjobid":"2e28660c-a971-4d93-b4a6-5e6256b7b7d3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#359 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787315699467685890 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=8c60e558a9356c5a25645eebaade802e946e45b6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/359/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-4tsns GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-03d44d992f53cff0b3bb7ab1ee8dd6585d62f1b48625706d61992f6495e84f7f NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 ap_tiflow_release-7_5_pull_cdc_integration_mysql_test_359-4tsns GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=359 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] // withEnv "cluster-id": "default" "id": "c3ef0048-536a-4a2c-b48f-21f4de92d8de", "cluster-id": "default" run task successfully Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} Update changefeed config successfully! ID: test Info: {"upstream_id":7365720886656298615,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2024-05-06T11:19:14.444998603+08:00","start_ts":449567930270089221,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567932957065219,"checkpoint_ts":449567930728841258,"checkpoint_time":"2024-05-06 11:19:16.064"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 20-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:28 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/189f54be-e269-4f29-bd1a-1a4fe8630770 {"id":"189f54be-e269-4f29-bd1a-1a4fe8630770","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be98a6829 189f54be-e269-4f29-bd1a-1a4fe8630770 /tidb/cdc/default/default/upstream/7365720951119582236 {"id":7365720951119582236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/189f54be-e269-4f29-bd1a-1a4fe8630770 {"id":"189f54be-e269-4f29-bd1a-1a4fe8630770","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be98a6829 189f54be-e269-4f29-bd1a-1a4fe8630770 /tidb/cdc/default/default/upstream/7365720951119582236 {"id":7365720951119582236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/189f54be-e269-4f29-bd1a-1a4fe8630770 {"id":"189f54be-e269-4f29-bd1a-1a4fe8630770","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be98a6829 189f54be-e269-4f29-bd1a-1a4fe8630770 /tidb/cdc/default/default/upstream/7365720951119582236 {"id":7365720951119582236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5136.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: e3ce8e62-ed50-48f7-b596-4b1c9b078f9d Info: {"upstream_id":7365720951119582236,"namespace":"default","id":"e3ce8e62-ed50-48f7-b596-4b1c9b078f9d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:19:28.650384625+08:00","start_ts":449567933997252609,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567933997252609,"checkpoint_ts":449567933997252609,"checkpoint_time":"2024-05-06 11:19:28.532"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 11:19:29 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.54115413.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 capture_id: c3ef0048-536a-4a2c-b48f-21f4de92d8de ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5305.out cli changefeed resume -c test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x table batch_add_table.finish_mark not exists for 1-th check, retry later table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567931937062938, "checkpoint_time": "2024-05-06 11:19:20.673", "error": null }' + echo '{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567931937062938, "checkpoint_time": "2024-05-06 11:19:20.673", "error": null }' { "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567931937062938, "checkpoint_time": "2024-05-06 11:19:20.673", "error": null } ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449567931937062938, '"checkpoint_time":' '"2024-05-06' '11:19:20.673",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449567931937062938, '"checkpoint_time":' '"2024-05-06' '11:19:20.673",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 The 1 times to try to start tidb cluster... changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11105.out cli changefeed --changefeed-id custom-changefeed-name pause ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 21-th time, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table batch_update_to_no_batch.v1 not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2c0161c-9a7b-430a-96ae-dcdfea5d655a {"id":"c2c0161c-9a7b-430a-96ae-dcdfea5d655a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be99938f5 c2c0161c-9a7b-430a-96ae-dcdfea5d655a /tidb/cdc/default/default/upstream/7365720960253340388 {"id":7365720960253340388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2c0161c-9a7b-430a-96ae-dcdfea5d655a {"id":"c2c0161c-9a7b-430a-96ae-dcdfea5d655a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be99938f5 c2c0161c-9a7b-430a-96ae-dcdfea5d655a /tidb/cdc/default/default/upstream/7365720960253340388 {"id":7365720960253340388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c2c0161c-9a7b-430a-96ae-dcdfea5d655a {"id":"c2c0161c-9a7b-430a-96ae-dcdfea5d655a","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be99938f5 c2c0161c-9a7b-430a-96ae-dcdfea5d655a /tidb/cdc/default/default/upstream/7365720960253340388 {"id":7365720960253340388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 [Mon May 6 11:19:32 CST 2024] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36403642.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 *************************** 1. row *************************** count(distinct region_id): 3 + set +x start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7a990000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:9247, start at 2024-05-06 11:19:32.216564386 +0800 CST m=+6.595915986 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:32.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:32.246 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:32.246 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7a990000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:9247, start at 2024-05-06 11:19:32.216564386 +0800 CST m=+6.595915986 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:32.231 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:32.246 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:32.246 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7abe40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-h8lrf, pid:9311, start at 2024-05-06 11:19:32.375230892 +0800 CST m=+6.702936050 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:32.383 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:32.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:32.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Starting Downstream PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 Verifying upstream PD is started... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030ed200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567934015340545 MinTableBarrierTs:449567934015340548 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/dc5fc4b1-21f3-4555-b9a5-2455188b3a05 {"id":"dc5fc4b1-21f3-4555-b9a5-2455188b3a05","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0870c dc5fc4b1-21f3-4555-b9a5-2455188b3a05 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567934015340545,"min-table-barrier-ts":449567934015340548,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dc5fc4b1-21f3-4555-b9a5-2455188b3a05/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030ed200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567934015340545 MinTableBarrierTs:449567934015340548 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/dc5fc4b1-21f3-4555-b9a5-2455188b3a05 {"id":"dc5fc4b1-21f3-4555-b9a5-2455188b3a05","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0870c dc5fc4b1-21f3-4555-b9a5-2455188b3a05 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567934015340545,"min-table-barrier-ts":449567934015340548,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dc5fc4b1-21f3-4555-b9a5-2455188b3a05/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/896d0161-e516-48a6-84de-2b5d7c054965 {UpstreamID:7365720710740210451 Namespace:default ID:896d0161-e516-48a6-84de-2b5d7c054965 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2024-05-06 11:18:33.35996629 +0800 CST StartTs:449567918155628545 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0030ed200 State:normal Error: Warning: CreatorVersion:v7.5.1-26-g93530e277 Epoch:449567919531884546} {CheckpointTs:449567934015340545 MinTableBarrierTs:449567934015340548 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c3ef0048-536a-4a2c-b48f-21f4de92d8de {"id":"c3ef0048-536a-4a2c-b48f-21f4de92d8de","address":"127.0.0.1:8301","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/dc5fc4b1-21f3-4555-b9a5-2455188b3a05 {"id":"dc5fc4b1-21f3-4555-b9a5-2455188b3a05","address":"127.0.0.1:8302","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/capture/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c {"id":"f974e8e4-15a5-4c5d-9c4a-3e9056e8818c","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b08696 f974e8e4-15a5-4c5d-9c4a-3e9056e8818c /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b086d6 c3ef0048-536a-4a2c-b48f-21f4de92d8de /tidb/cdc/default/__cdc_meta__/owner/22318f4be8b0870c dc5fc4b1-21f3-4555-b9a5-2455188b3a05 /tidb/cdc/default/default/changefeed/info/896d0161-e516-48a6-84de-2b5d7c054965 {"upstream-id":7365720710740210451,"namespace":"default","changefeed-id":"896d0161-e516-48a6-84de-2b5d7c054965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2024-05-06T11:18:33.35996629+08:00","start-ts":449567918155628545,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"open":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-26-g93530e277","epoch":449567919531884546} /tidb/cdc/default/default/changefeed/status/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":449567934015340545,"min-table-barrier-ts":449567934015340548,"admin-job-type":0} /tidb/cdc/default/default/task/position/c3ef0048-536a-4a2c-b48f-21f4de92d8de/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dc5fc4b1-21f3-4555-b9a5-2455188b3a05/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f974e8e4-15a5-4c5d-9c4a-3e9056e8818c/896d0161-e516-48a6-84de-2b5d7c054965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365720710740210451 {"id":7365720710740210451,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' wait process cdc.test exit for 3-th time... run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:19:35 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> [Mon May 6 11:19:35 CST 2024] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1057310575.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 4-th time... table batch_update_to_no_batch.recover_and_insert not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449567934532026375, "checkpoint_time": "2024-05-06 11:19:30.572", "error": null }' + echo '{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449567934532026375, "checkpoint_time": "2024-05-06 11:19:30.572", "error": null }' { "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449567934532026375, "checkpoint_time": "2024-05-06 11:19:30.572", "error": null } ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449567934532026375, '"checkpoint_time":' '"2024-05-06' '11:19:30.572",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449567934532026375, '"checkpoint_time":' '"2024-05-06' '11:19:30.572",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11169.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name cdc.test: no process found wait process cdc.test exit for 5-th time... process cdc.test already exit [Mon May 6 11:19:36 CST 2024] <<<<<< run test case clustered_index success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365720931959301550,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T11:19:24.276942391+08:00","start_ts":449567931517632513,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-26-g93530e277","resolved_ts":449567934532026375,"checkpoint_ts":449567934532026375,"checkpoint_time":"2024-05-06 11:19:30.572"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... split_and_random_merge scale: 40 [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7cd3c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:10125, start at 2024-05-06 11:19:34.517711278 +0800 CST m=+5.366653571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:34.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:34.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:34.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7cd3c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:10125, start at 2024-05-06 11:19:34.517711278 +0800 CST m=+5.366653571 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:34.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:34.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:34.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7cf24000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-1hsdj, pid:10196, start at 2024-05-06 11:19:34.619287443 +0800 CST m=+5.394035736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:34.627 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:34.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:34.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7f9ec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:7501, start at 2024-05-06 11:19:37.359368953 +0800 CST m=+5.405827874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:37.367 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:37.339 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:37.339 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task failed 2-th time, retry later + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11201.out cli changefeed --changefeed-id custom-changefeed-name resume [Pipeline] // node table batch_update_to_no_batch.recover_and_insert not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 03:19:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 {"id":"1d0a7b64-b7b2-471f-9fae-99d5a9678fa4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be9b8c1f4 1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 /tidb/cdc/default/default/upstream/7365720994191147040 {"id":7365720994191147040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 {"id":"1d0a7b64-b7b2-471f-9fae-99d5a9678fa4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be9b8c1f4 1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 /tidb/cdc/default/default/upstream/7365720994191147040 {"id":7365720994191147040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 {"id":"1d0a7b64-b7b2-471f-9fae-99d5a9678fa4","address":"127.0.0.1:8300","version":"v7.5.1-26-g93530e277"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4be9b8c1f4 1d0a7b64-b7b2-471f-9fae-99d5a9678fa4 /tidb/cdc/default/default/upstream/7365720994191147040 {"id":7365720994191147040,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.11579.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7f9ec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:7501, start at 2024-05-06 11:19:37.359368953 +0800 CST m=+5.405827874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:37.367 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:37.339 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:37.339 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa7fab4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-w7hmn, pid:7574, start at 2024-05-06 11:19:37.421519225 +0800 CST m=+5.393435764 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:37.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:37.439 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:37.439 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-47-g61c4bd6125 Edition: Community Git Commit Hash: 61c4bd61254323d2dc68a8de048000ea036e67ac Git Branch: release-7.5 UTC Build Time: 2024-05-01 02:52:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later ***************** properties ***************** "threadcount"="10" "updateproportion"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "readproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "mysql.db"="consistent_replicate_storage_file" "workload"="core" "readallfields"="true" "recordcount"="5000" "scanproportion"="0" "mysql.port"="4000" "operationcount"="0" "fieldcount"="100" "insertproportion"="0" ********************************************** + set +x [Pipeline] // node [Pipeline] } [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449567936958693377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567936958693377 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later [Pipeline] // podTemplate [Mon May 6 11:19:42 CST 2024] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1162311625.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8830.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // podTemplate [Pipeline] } cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G01' [Pipeline] // withEnv [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 2.40826844s INSERT - Takes(s): 2.4, Count: 5000, OPS: 2084.0, Avg(us): 4463, Min(us): 1337, Max(us): 146022, 95th(us): 6000, 99th(us): 58000 [Pipeline] } Sending interrupt signal to process Killing processes ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes *************************** 1. row *************************** count(distinct region_id): 1 table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567937690861571, "checkpoint_time": "2024-05-06 11:19:42.622", "error": null }' + echo '{ "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567937690861571, "checkpoint_time": "2024-05-06 11:19:42.622", "error": null }' { "upstream_id": 7365720931959301550, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449567937690861571, "checkpoint_time": "2024-05-06 11:19:42.622", "error": null } ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449567937690861571, '"checkpoint_time":' '"2024-05-06' '11:19:42.622",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365720931959301550, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449567937690861571, '"checkpoint_time":' '"2024-05-06' '11:19:42.622",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11267.out cli changefeed --changefeed-id custom-changefeed-name remove {"level":"warn","ts":1714965583.6991234,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00240fa40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes + set +x + tso='449567937695580161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449567937695580161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 11:19:44 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.88708872.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714965584.523321,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002042700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = closing transport due to: connection error: desc = \"error reading from server: EOF\", received prior goaway: code: NO_ERROR, debug data: \"graceful_stop\""} script returned exit code 143 table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449567937953005572 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714965584.797288,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0027ce380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... kill finished with exit code 0 Sending interrupt signal to process Killing processes VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa88778000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr, pid:1193, start at 2024-05-06 11:19:46.419069724 +0800 CST m=+5.908187035 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:46.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:46.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:46.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later {"level":"warn","ts":1714965586.5137167,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012f3dc0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa88778000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr, pid:1193, start at 2024-05-06 11:19:46.419069724 +0800 CST m=+5.908187035 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:46.428 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:46.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:46.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa88aa00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-flnrr, pid:1233, start at 2024-05-06 11:19:46.60088663 +0800 CST m=+6.000343662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:46.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:46.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:46.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table batch_update_to_no_batch.finish_mark exists check diff successfully Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714965589.4192286,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00268a540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa898c00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81, pid:1169, start at 2024-05-06 11:19:47.513786422 +0800 CST m=+8.645268132 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:47.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:47.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:47.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa898c00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81, pid:1169, start at 2024-05-06 11:19:47.513786422 +0800 CST m=+8.645268132 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:47.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:47.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:47.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d2fa89c74000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-mysql-test-359-c7n81, pid:1203, start at 2024-05-06 11:19:47.778846398 +0800 CST m=+8.750130904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-11:21:47.799 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-11:19:47.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-11:09:47.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } kill finished with exit code 0 Sending interrupt signal to process Killing processes wait process cdc.test exit for 2-th time... empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 11:19:50 CST 2024] <<<<<< run test case batch_update_to_no_batch success! >>>>>> script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 {"level":"warn","ts":1714965590.5296798,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0023dc8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 [Pipeline] // stage [Pipeline] } script returned exit code 143 [Pipeline] // stage script returned exit code 143 {"level":"warn","ts":1714965586.0182388,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f85500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache script returned exit code 143 [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh: line 1: 2767 Terminated GO111MODULE=on go run main.go -config ./config.toml 2>&1 2768 | tee $WORK_DIR/tester.log script returned exit code 143 [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // node [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G17' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G18' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G19' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G20' [Pipeline] // stage [Pipeline] // node [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G21' [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] // stage [Pipeline] // podTemplate [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE